You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/01/10 12:26:41 UTC
[1/5] tajo git commit: TAJO-1281: Remove hadoop-common dependency
from tajo-rpc.
Repository: tajo
Updated Branches:
refs/heads/index_support 767b9a4b7 -> ad78d2f62
TAJO-1281: Remove hadoop-common dependency from tajo-rpc.
Closes #343
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/9eac34fe
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/9eac34fe
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/9eac34fe
Branch: refs/heads/index_support
Commit: 9eac34fe3121b8bc6bbe1b81ca76852ddedb2603
Parents: 807868b
Author: Hyunsik Choi <hy...@apache.org>
Authored: Sat Jan 10 16:12:08 2015 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Sat Jan 10 16:12:08 2015 +0900
----------------------------------------------------------------------
CHANGES | 2 +
.../tajo/catalog/AbstractCatalogClient.java | 11 +---
.../apache/tajo/client/SessionConnection.java | 2 +-
.../org/apache/tajo/master/QueryInProgress.java | 4 +-
.../apache/tajo/master/TajoContainerProxy.java | 14 ++--
.../apache/tajo/querymaster/QueryMaster.java | 2 +-
.../tajo/worker/ExecutionBlockContext.java | 2 +-
.../tajo/worker/TajoResourceAllocator.java | 6 +-
.../tajo/worker/WorkerHeartbeatService.java | 2 +-
.../ConnectivityCheckerRuleForTajoWorker.java | 2 +-
tajo-rpc/pom.xml | 27 ++++----
.../org/apache/tajo/rpc/AsyncRpcClient.java | 3 +-
.../org/apache/tajo/rpc/BlockingRpcClient.java | 5 +-
.../org/apache/tajo/rpc/NettyClientBase.java | 3 +-
.../org/apache/tajo/rpc/NettyServerBase.java | 4 +-
.../org/apache/tajo/rpc/RpcChannelFactory.java | 18 ++++--
.../org/apache/tajo/rpc/RpcConnectionPool.java | 13 ++--
.../main/java/org/apache/tajo/rpc/RpcUtils.java | 68 ++++++++++++++++++++
.../org/apache/tajo/rpc/ServerCallable.java | 5 +-
.../java/org/apache/tajo/rpc/TestAsyncRpc.java | 7 +-
.../org/apache/tajo/rpc/TestBlockingRpc.java | 12 ++--
21 files changed, 137 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 89488da..b5578e7 100644
--- a/CHANGES
+++ b/CHANGES
@@ -27,6 +27,8 @@ Release 0.9.1 - unreleased
IMPROVEMENT
+ TAJO-1281: Remove hadoop-common dependency from tajo-rpc. (hyunsik)
+
TAJO-1291: Rename TajoMasterProtocol to QueryCoordinatorProtocol.
(hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
index 8ef1c9a..1a2fd44 100644
--- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
+++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
@@ -19,7 +19,6 @@
package org.apache.tajo.catalog;
import com.google.protobuf.ServiceException;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tajo.annotation.Nullable;
@@ -27,21 +26,15 @@ import org.apache.tajo.catalog.CatalogProtocol.CatalogProtocolService;
import org.apache.tajo.catalog.exception.NoSuchFunctionException;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
-import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.DatabaseProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableDescriptorProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableOptionProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto;
-import org.apache.tajo.catalog.proto.CatalogProtos.TablespaceProto;
import org.apache.tajo.catalog.proto.CatalogProtos.*;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.ha.HAServiceUtil;
import org.apache.tajo.rpc.NettyClientBase;
import org.apache.tajo.rpc.RpcConnectionPool;
import org.apache.tajo.rpc.ServerCallable;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.NullProto;
-import org.apache.tajo.ha.HAServiceUtil;
import org.apache.tajo.util.ProtoUtil;
import java.net.InetSocketAddress;
@@ -62,7 +55,7 @@ public abstract class AbstractCatalogClient implements CatalogService {
abstract CatalogProtocolService.BlockingInterface getStub(NettyClientBase client);
public AbstractCatalogClient(TajoConf conf, InetSocketAddress catalogServerAddr) {
- this.pool = RpcConnectionPool.getPool(conf);
+ this.pool = RpcConnectionPool.getPool();
this.catalogServerAddr = catalogServerAddr;
this.conf = conf;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java
index 1bc8050..5490be4 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java
@@ -105,7 +105,7 @@ public class SessionConnection implements Closeable {
this.tajoMasterAddr = addr;
int workerNum = conf.getIntVar(TajoConf.ConfVars.RPC_CLIENT_WORKER_THREAD_NUM);
// Don't share connection pool per client
- connPool = RpcConnectionPool.newPool(conf, getClass().getSimpleName(), workerNum);
+ connPool = RpcConnectionPool.newPool(getClass().getSimpleName(), workerNum);
userInfo = UserRoleInfo.getCurrentUser();
this.baseDatabase = baseDatabase != null ? baseDatabase : null;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java b/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java
index 73d8cb2..7587543 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/QueryInProgress.java
@@ -96,7 +96,7 @@ public class QueryInProgress {
masterContext.getResourceManager().releaseQueryMaster(queryId);
if(queryMasterRpc != null) {
- RpcConnectionPool.getPool(masterContext.getConf()).closeConnection(queryMasterRpc);
+ RpcConnectionPool.getPool().closeConnection(queryMasterRpc);
}
masterContext.getHistoryWriter().appendHistory(queryInfo);
@@ -130,7 +130,7 @@ public class QueryInProgress {
InetSocketAddress addr = NetUtils.createSocketAddr(queryInfo.getQueryMasterHost(), queryInfo.getQueryMasterPort());
LOG.info("Connect to QueryMaster:" + addr);
queryMasterRpc =
- RpcConnectionPool.getPool(masterContext.getConf()).getConnection(addr, QueryMasterProtocol.class, true);
+ RpcConnectionPool.getPool().getConnection(addr, QueryMasterProtocol.class, true);
queryMasterRpcClient = queryMasterRpc.getStub();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java
index 2ffd7ca..588b7ee 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java
@@ -25,18 +25,18 @@ import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.TaskAttemptId;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.query.QueryContext;
+import org.apache.tajo.ha.HAServiceUtil;
import org.apache.tajo.ipc.ContainerProtocol;
import org.apache.tajo.ipc.QueryCoordinatorProtocol;
import org.apache.tajo.ipc.TajoWorkerProtocol;
-import org.apache.tajo.querymaster.QueryMasterTask;
import org.apache.tajo.master.container.TajoContainer;
import org.apache.tajo.master.container.TajoContainerId;
import org.apache.tajo.master.rm.TajoWorkerContainer;
import org.apache.tajo.master.rm.TajoWorkerContainerId;
+import org.apache.tajo.querymaster.QueryMasterTask;
import org.apache.tajo.rpc.NettyClientBase;
import org.apache.tajo.rpc.NullCallback;
import org.apache.tajo.rpc.RpcConnectionPool;
-import org.apache.tajo.ha.HAServiceUtil;
import java.net.InetSocketAddress;
import java.util.ArrayList;
@@ -79,13 +79,13 @@ public class TajoContainerProxy extends ContainerProxy {
NettyClientBase tajoWorkerRpc = null;
try {
InetSocketAddress addr = new InetSocketAddress(container.getNodeId().getHost(), container.getNodeId().getPort());
- tajoWorkerRpc = RpcConnectionPool.getPool(context.getConf()).getConnection(addr, TajoWorkerProtocol.class, true);
+ tajoWorkerRpc = RpcConnectionPool.getPool().getConnection(addr, TajoWorkerProtocol.class, true);
TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub();
tajoWorkerRpcClient.killTaskAttempt(null, taskAttemptId.getProto(), NullCallback.get());
} catch (Exception e) {
LOG.error(e.getMessage(), e);
} finally {
- RpcConnectionPool.getPool(context.getConf()).releaseConnection(tajoWorkerRpc);
+ RpcConnectionPool.getPool().releaseConnection(tajoWorkerRpc);
}
}
@@ -96,7 +96,7 @@ public class TajoContainerProxy extends ContainerProxy {
.getQueryMasterManagerService().getBindAddr();
InetSocketAddress addr = new InetSocketAddress(container.getNodeId().getHost(), container.getNodeId().getPort());
- tajoWorkerRpc = RpcConnectionPool.getPool(context.getConf()).getConnection(addr, TajoWorkerProtocol.class, true);
+ tajoWorkerRpc = RpcConnectionPool.getPool().getConnection(addr, TajoWorkerProtocol.class, true);
TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub();
TajoWorkerProtocol.RunExecutionBlockRequestProto request =
@@ -114,7 +114,7 @@ public class TajoContainerProxy extends ContainerProxy {
} catch (Exception e) {
LOG.error(e.getMessage(), e);
} finally {
- RpcConnectionPool.getPool(context.getConf()).releaseConnection(tajoWorkerRpc);
+ RpcConnectionPool.getPool().releaseConnection(tajoWorkerRpc);
}
}
@@ -166,7 +166,7 @@ public class TajoContainerProxy extends ContainerProxy {
containerIdProtos.add(TajoWorkerContainerId.getContainerIdProto(eachContainerId));
}
- RpcConnectionPool connPool = RpcConnectionPool.getPool(context.getConf());
+ RpcConnectionPool connPool = RpcConnectionPool.getPool();
NettyClientBase tmClient = null;
try {
// In TajoMaster HA mode, if backup master be active status,
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java
index 53390a1..be78fc3 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMaster.java
@@ -101,7 +101,7 @@ public class QueryMaster extends CompositeService implements EventHandler {
LOG.info("QueryMaster init");
try {
this.systemConf = (TajoConf)conf;
- this.connPool = RpcConnectionPool.getPool(systemConf);
+ this.connPool = RpcConnectionPool.getPool();
querySessionTimeout = systemConf.getIntVar(TajoConf.ConfVars.QUERY_SESSION_TIMEOUT);
queryMasterContext = new QueryMasterContext(systemConf);
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java
index dd3ee68..b120d5b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockContext.java
@@ -100,7 +100,7 @@ public class ExecutionBlockContext {
throws Throwable {
this.manager = manager;
this.executionBlockId = event.getExecutionBlockId();
- this.connPool = RpcConnectionPool.getPool(manager.getTajoConf());
+ this.connPool = RpcConnectionPool.getPool();
this.queryMaster = queryMaster;
this.systemConf = manager.getTajoConf();
this.reporter = new Reporter();
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/tajo-core/src/main/java/org/apache/tajo/worker/TajoResourceAllocator.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoResourceAllocator.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoResourceAllocator.java
index 04b65d2..7278317 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoResourceAllocator.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoResourceAllocator.java
@@ -186,14 +186,14 @@ public class TajoResourceAllocator extends AbstractResourceAllocator {
NettyClientBase tajoWorkerRpc = null;
try {
InetSocketAddress addr = new InetSocketAddress(worker.getHost(), worker.getPort());
- tajoWorkerRpc = RpcConnectionPool.getPool(tajoConf).getConnection(addr, TajoWorkerProtocol.class, true);
+ tajoWorkerRpc = RpcConnectionPool.getPool().getConnection(addr, TajoWorkerProtocol.class, true);
TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub();
tajoWorkerRpcClient.stopExecutionBlock(null, executionBlockId.getProto(), NullCallback.get());
} catch (Throwable e) {
LOG.error(e.getMessage(), e);
} finally {
- RpcConnectionPool.getPool(tajoConf).releaseConnection(tajoWorkerRpc);
+ RpcConnectionPool.getPool().releaseConnection(tajoWorkerRpc);
}
}
@@ -271,7 +271,7 @@ public class TajoResourceAllocator extends AbstractResourceAllocator {
.setQueryId(event.getExecutionBlockId().getQueryId().getProto())
.build();
- RpcConnectionPool connPool = RpcConnectionPool.getPool(queryTaskContext.getConf());
+ RpcConnectionPool connPool = RpcConnectionPool.getPool();
NettyClientBase tmClient = null;
try {
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java b/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java
index b92c4cd..676c72b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java
@@ -74,7 +74,7 @@ public class WorkerHeartbeatService extends AbstractService {
Preconditions.checkArgument(conf instanceof TajoConf, "Configuration must be a TajoConf instance.");
this.systemConf = (TajoConf) conf;
- connectionPool = RpcConnectionPool.getPool(systemConf);
+ connectionPool = RpcConnectionPool.getPool();
super.serviceInit(conf);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java
index 68890e3..9d9f39c 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java
@@ -40,7 +40,7 @@ import java.net.InetSocketAddress;
public class ConnectivityCheckerRuleForTajoWorker implements SelfDiagnosisRule {
private void checkTajoMasterConnectivity(TajoConf tajoConf) throws Exception {
- RpcConnectionPool pool = RpcConnectionPool.getPool(tajoConf);
+ RpcConnectionPool pool = RpcConnectionPool.getPool();
NettyClientBase masterClient = null;
InetSocketAddress masterAddress = null;
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/tajo-rpc/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-rpc/pom.xml b/tajo-rpc/pom.xml
index 1e00e70..d0037ca 100644
--- a/tajo-rpc/pom.xml
+++ b/tajo-rpc/pom.xml
@@ -137,23 +137,24 @@
<dependencies>
<dependency>
- <groupId>org.apache.tajo</groupId>
- <artifactId>tajo-common</artifactId>
- <exclusions>
- <exclusion>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/tajo-rpc/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java
----------------------------------------------------------------------
diff --git a/tajo-rpc/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java b/tajo-rpc/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java
index 7a416a8..4b1842e 100644
--- a/tajo-rpc/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java
+++ b/tajo-rpc/src/main/java/org/apache/tajo/rpc/AsyncRpcClient.java
@@ -24,7 +24,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tajo.rpc.RpcProtos.RpcRequest;
import org.apache.tajo.rpc.RpcProtos.RpcResponse;
-import org.apache.tajo.util.NetUtils;
import org.jboss.netty.channel.*;
import org.jboss.netty.channel.socket.ClientSocketChannelFactory;
@@ -180,7 +179,7 @@ public class AsyncRpcClient extends NettyClientBase {
private String getErrorMessage(String message) {
return "Exception [" + protocol.getCanonicalName() +
- "(" + NetUtils.normalizeInetSocketAddress((InetSocketAddress)
+ "(" + RpcUtils.normalizeInetSocketAddress((InetSocketAddress)
getChannel().getRemoteAddress()) + ")]: " + message;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/tajo-rpc/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java
----------------------------------------------------------------------
diff --git a/tajo-rpc/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java b/tajo-rpc/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java
index 03d5d3e..869919c 100644
--- a/tajo-rpc/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java
+++ b/tajo-rpc/src/main/java/org/apache/tajo/rpc/BlockingRpcClient.java
@@ -27,7 +27,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tajo.rpc.RpcProtos.RpcRequest;
import org.apache.tajo.rpc.RpcProtos.RpcResponse;
-import org.apache.tajo.util.NetUtils;
import org.jboss.netty.channel.*;
import org.jboss.netty.channel.socket.ClientSocketChannelFactory;
@@ -159,7 +158,7 @@ public class BlockingRpcClient extends NettyClientBase {
private String getErrorMessage(String message) {
if(protocol != null && getChannel() != null) {
return protocol.getName() +
- "(" + NetUtils.normalizeInetSocketAddress((InetSocketAddress)
+ "(" + RpcUtils.normalizeInetSocketAddress((InetSocketAddress)
getChannel().getRemoteAddress()) + "): " + message;
} else {
return "Exception " + message;
@@ -169,7 +168,7 @@ public class BlockingRpcClient extends NettyClientBase {
private TajoServiceException makeTajoServiceException(RpcResponse response, Throwable cause) {
if(protocol != null && getChannel() != null) {
return new TajoServiceException(response.getErrorMessage(), cause, protocol.getName(),
- NetUtils.normalizeInetSocketAddress((InetSocketAddress)getChannel().getRemoteAddress()));
+ RpcUtils.normalizeInetSocketAddress((InetSocketAddress)getChannel().getRemoteAddress()));
} else {
return new TajoServiceException(response.getErrorMessage());
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/tajo-rpc/src/main/java/org/apache/tajo/rpc/NettyClientBase.java
----------------------------------------------------------------------
diff --git a/tajo-rpc/src/main/java/org/apache/tajo/rpc/NettyClientBase.java b/tajo-rpc/src/main/java/org/apache/tajo/rpc/NettyClientBase.java
index d0002de..bc0c567 100644
--- a/tajo-rpc/src/main/java/org/apache/tajo/rpc/NettyClientBase.java
+++ b/tajo-rpc/src/main/java/org/apache/tajo/rpc/NettyClientBase.java
@@ -21,7 +21,6 @@ package org.apache.tajo.rpc;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.tajo.util.NetUtils;
import org.jboss.netty.bootstrap.ClientBootstrap;
import org.jboss.netty.channel.*;
import org.jboss.netty.channel.socket.ClientSocketChannelFactory;
@@ -110,7 +109,7 @@ public abstract class NettyClientBase implements Closeable {
public void connect(InetSocketAddress addr) throws ConnectTimeoutException {
if(addr.isUnresolved()){
- addr = NetUtils.createSocketAddr(addr.getHostName(), addr.getPort());
+ addr = RpcUtils.createSocketAddr(addr.getHostName(), addr.getPort());
}
handleConnectionInternally(addr);
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/tajo-rpc/src/main/java/org/apache/tajo/rpc/NettyServerBase.java
----------------------------------------------------------------------
diff --git a/tajo-rpc/src/main/java/org/apache/tajo/rpc/NettyServerBase.java b/tajo-rpc/src/main/java/org/apache/tajo/rpc/NettyServerBase.java
index e75418d..ef090ff 100644
--- a/tajo-rpc/src/main/java/org/apache/tajo/rpc/NettyServerBase.java
+++ b/tajo-rpc/src/main/java/org/apache/tajo/rpc/NettyServerBase.java
@@ -20,12 +20,10 @@ package org.apache.tajo.rpc;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.tajo.util.NetUtils;
import org.jboss.netty.bootstrap.ServerBootstrap;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelFactory;
import org.jboss.netty.channel.ChannelPipelineFactory;
-import org.jboss.netty.channel.DefaultChannelFuture;
import org.jboss.netty.channel.group.ChannelGroup;
import org.jboss.netty.channel.group.DefaultChannelGroup;
@@ -126,7 +124,7 @@ public class NettyServerBase {
if (bindAddress != null) {
LOG.info("Rpc (" + serviceName + ") listened on "
- + NetUtils.normalizeInetSocketAddress(bindAddress)+ ") shutdown");
+ + RpcUtils.normalizeInetSocketAddress(bindAddress)+ ") shutdown");
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/tajo-rpc/src/main/java/org/apache/tajo/rpc/RpcChannelFactory.java
----------------------------------------------------------------------
diff --git a/tajo-rpc/src/main/java/org/apache/tajo/rpc/RpcChannelFactory.java b/tajo-rpc/src/main/java/org/apache/tajo/rpc/RpcChannelFactory.java
index 6274eff..0727f71 100644
--- a/tajo-rpc/src/main/java/org/apache/tajo/rpc/RpcChannelFactory.java
+++ b/tajo-rpc/src/main/java/org/apache/tajo/rpc/RpcChannelFactory.java
@@ -21,7 +21,6 @@ package org.apache.tajo.rpc;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.tajo.conf.TajoConf;
import org.jboss.netty.channel.socket.ClientSocketChannelFactory;
import org.jboss.netty.channel.socket.ServerSocketChannelFactory;
import org.jboss.netty.channel.socket.nio.*;
@@ -34,6 +33,9 @@ import java.util.concurrent.atomic.AtomicInteger;
public final class RpcChannelFactory {
private static final Log LOG = LogFactory.getLog(RpcChannelFactory.class);
+
+ private static final int DEFAULT_WORKER_NUM = Runtime.getRuntime().availableProcessors() * 2;
+
private static ClientSocketChannelFactory factory;
private static AtomicInteger clientCount = new AtomicInteger(0);
private static AtomicInteger serverCount = new AtomicInteger(0);
@@ -45,11 +47,19 @@ public final class RpcChannelFactory {
* make this factory static thus all clients can share its thread pool.
* NioClientSocketChannelFactory has only one method newChannel() visible for user, which is thread-safe
*/
- public static synchronized ClientSocketChannelFactory getSharedClientChannelFactory(){
+ public static synchronized ClientSocketChannelFactory getSharedClientChannelFactory() {
+ return getSharedClientChannelFactory(DEFAULT_WORKER_NUM);
+ }
+
+ /**
+ * make this factory static thus all clients can share its thread pool.
+ * NioClientSocketChannelFactory has only one method newChannel() visible for user, which is thread-safe
+ *
+ * @param workerNum The number of workers
+ */
+ public static synchronized ClientSocketChannelFactory getSharedClientChannelFactory(int workerNum){
//shared woker and boss pool
if(factory == null){
- TajoConf conf = new TajoConf();
- int workerNum = conf.getIntVar(TajoConf.ConfVars.INTERNAL_RPC_CLIENT_WORKER_THREAD_NUM);
factory = createClientChannelFactory("Internal-Client", workerNum);
}
return factory;
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/tajo-rpc/src/main/java/org/apache/tajo/rpc/RpcConnectionPool.java
----------------------------------------------------------------------
diff --git a/tajo-rpc/src/main/java/org/apache/tajo/rpc/RpcConnectionPool.java b/tajo-rpc/src/main/java/org/apache/tajo/rpc/RpcConnectionPool.java
index 2f3d433..c8e622b 100644
--- a/tajo-rpc/src/main/java/org/apache/tajo/rpc/RpcConnectionPool.java
+++ b/tajo-rpc/src/main/java/org/apache/tajo/rpc/RpcConnectionPool.java
@@ -21,7 +21,6 @@ package org.apache.tajo.rpc;
import com.google.common.base.Objects;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.tajo.conf.TajoConf;
import org.jboss.netty.channel.ConnectTimeoutException;
import org.jboss.netty.channel.group.ChannelGroup;
import org.jboss.netty.channel.group.DefaultChannelGroup;
@@ -43,25 +42,23 @@ public class RpcConnectionPool {
private static RpcConnectionPool instance;
private final ClientSocketChannelFactory channelFactory;
- private final TajoConf conf;
public final static int RPC_RETRIES = 3;
- private RpcConnectionPool(TajoConf conf, ClientSocketChannelFactory channelFactory) {
- this.conf = conf;
+ private RpcConnectionPool(ClientSocketChannelFactory channelFactory) {
this.channelFactory = channelFactory;
}
- public synchronized static RpcConnectionPool getPool(TajoConf conf) {
+ public synchronized static RpcConnectionPool getPool() {
if(instance == null) {
InternalLoggerFactory.setDefaultFactory(new CommonsLoggerFactory());
- instance = new RpcConnectionPool(conf, RpcChannelFactory.getSharedClientChannelFactory());
+ instance = new RpcConnectionPool(RpcChannelFactory.getSharedClientChannelFactory());
}
return instance;
}
- public synchronized static RpcConnectionPool newPool(TajoConf conf, String poolName, int workerNum) {
- return new RpcConnectionPool(conf, RpcChannelFactory.createClientChannelFactory(poolName, workerNum));
+ public synchronized static RpcConnectionPool newPool(String poolName, int workerNum) {
+ return new RpcConnectionPool(RpcChannelFactory.createClientChannelFactory(poolName, workerNum));
}
private NettyClientBase makeConnection(RpcConnectionKey rpcConnectionKey)
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/tajo-rpc/src/main/java/org/apache/tajo/rpc/RpcUtils.java
----------------------------------------------------------------------
diff --git a/tajo-rpc/src/main/java/org/apache/tajo/rpc/RpcUtils.java b/tajo-rpc/src/main/java/org/apache/tajo/rpc/RpcUtils.java
new file mode 100644
index 0000000..b6be05f
--- /dev/null
+++ b/tajo-rpc/src/main/java/org/apache/tajo/rpc/RpcUtils.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.rpc;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+
+public class RpcUtils {
+
+ public static String normalizeInetSocketAddress(InetSocketAddress addr) {
+ return addr.getAddress().getHostAddress() + ":" + addr.getPort();
+ }
+
+ /**
+ * Util method to build socket addr from either:
+ * <host>
+ * <host>:<port>
+ * <fs>://<host>:<port>/<path>
+ */
+ public static InetSocketAddress createSocketAddr(String host, int port) {
+ return new InetSocketAddress(host, port);
+ }
+
+ /**
+ * Returns InetSocketAddress that a client can use to
+ * connect to the server. NettyServerBase.getListenerAddress() is not correct when
+ * the server binds to "0.0.0.0". This returns "hostname:port" of the server,
+ * or "127.0.0.1:port" when the getListenerAddress() returns "0.0.0.0:port".
+ *
+ * @param addr of a listener
+ * @return socket address that a client can use to connect to the server.
+ */
+ public static InetSocketAddress getConnectAddress(InetSocketAddress addr) {
+ if (!addr.isUnresolved() && addr.getAddress().isAnyLocalAddress()) {
+ try {
+ addr = new InetSocketAddress(InetAddress.getLocalHost(), addr.getPort());
+ } catch (UnknownHostException uhe) {
+ // shouldn't get here unless the host doesn't have a loopback iface
+ addr = new InetSocketAddress("127.0.0.1", addr.getPort());
+ }
+ }
+ InetSocketAddress canonicalAddress =
+ new InetSocketAddress(addr.getAddress().getCanonicalHostName(), addr.getPort());
+ return canonicalAddress;
+ }
+
+ public static InetSocketAddress createUnresolved(String addr) {
+ String [] splitted = addr.split(":");
+ return InetSocketAddress.createUnresolved(splitted[0], Integer.parseInt(splitted[1]));
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/tajo-rpc/src/main/java/org/apache/tajo/rpc/ServerCallable.java
----------------------------------------------------------------------
diff --git a/tajo-rpc/src/main/java/org/apache/tajo/rpc/ServerCallable.java b/tajo-rpc/src/main/java/org/apache/tajo/rpc/ServerCallable.java
index b4e5f9a..140f781 100644
--- a/tajo-rpc/src/main/java/org/apache/tajo/rpc/ServerCallable.java
+++ b/tajo-rpc/src/main/java/org/apache/tajo/rpc/ServerCallable.java
@@ -18,15 +18,14 @@
package org.apache.tajo.rpc;
+import com.google.protobuf.ServiceException;
+
import java.io.IOException;
import java.lang.reflect.UndeclaredThrowableException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
-import com.google.protobuf.ServiceException;
-import org.apache.tajo.conf.TajoConf;
-
public abstract class ServerCallable<T> {
protected InetSocketAddress addr;
protected long startTime;
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java
----------------------------------------------------------------------
diff --git a/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java b/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java
index 7c8246a..61a92bc 100644
--- a/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java
+++ b/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestAsyncRpc.java
@@ -27,7 +27,6 @@ import org.apache.tajo.rpc.test.TestProtos.EchoMessage;
import org.apache.tajo.rpc.test.TestProtos.SumRequest;
import org.apache.tajo.rpc.test.TestProtos.SumResponse;
import org.apache.tajo.rpc.test.impl.DummyProtocolAsyncImpl;
-import org.apache.tajo.util.NetUtils;
import org.jboss.netty.channel.ConnectTimeoutException;
import org.jboss.netty.channel.socket.ClientSocketChannelFactory;
import org.junit.After;
@@ -65,7 +64,7 @@ public class TestAsyncRpc {
service, new InetSocketAddress("127.0.0.1", 0), 2);
server.start();
client = new AsyncRpcClient(DummyProtocol.class,
- NetUtils.getConnectAddress(server.getListenAddress()), clientChannelFactory, retries);
+ RpcUtils.getConnectAddress(server.getListenAddress()), clientChannelFactory, retries);
stub = client.getStub();
}
@@ -256,9 +255,9 @@ public class TestAsyncRpc {
client.close();
client = null;
- String hostAndPort = NetUtils.normalizeInetSocketAddress(server.getListenAddress());
+ String hostAndPort = RpcUtils.normalizeInetSocketAddress(server.getListenAddress());
client = new AsyncRpcClient(DummyProtocol.class,
- NetUtils.createUnresolved(hostAndPort), clientChannelFactory, retries);
+ RpcUtils.createUnresolved(hostAndPort), clientChannelFactory, retries);
Interface stub = client.getStub();
EchoMessage echoMessage = EchoMessage.newBuilder()
.setMessage(MESSAGE).build();
http://git-wip-us.apache.org/repos/asf/tajo/blob/9eac34fe/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java
----------------------------------------------------------------------
diff --git a/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java b/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java
index 28a3fad..746bfcb 100644
--- a/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java
+++ b/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestBlockingRpc.java
@@ -18,14 +18,12 @@
package org.apache.tajo.rpc;
-import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.rpc.test.DummyProtocol;
import org.apache.tajo.rpc.test.DummyProtocol.DummyProtocolService.BlockingInterface;
import org.apache.tajo.rpc.test.TestProtos.EchoMessage;
import org.apache.tajo.rpc.test.TestProtos.SumRequest;
import org.apache.tajo.rpc.test.TestProtos.SumResponse;
import org.apache.tajo.rpc.test.impl.DummyProtocolBlockingImpl;
-import org.apache.tajo.util.NetUtils;
import org.jboss.netty.channel.socket.ClientSocketChannelFactory;
import org.junit.After;
import org.junit.Before;
@@ -59,7 +57,7 @@ public class TestBlockingRpc {
new InetSocketAddress("127.0.0.1", 0), 2);
server.start();
client = new BlockingRpcClient(DummyProtocol.class,
- NetUtils.getConnectAddress(server.getListenAddress()), clientChannelFactory, retries);
+ RpcUtils.getConnectAddress(server.getListenAddress()), clientChannelFactory, retries);
stub = client.getStub();
}
@@ -96,7 +94,7 @@ public class TestBlockingRpc {
@Test
public void testRpcWithServiceCallable() throws Exception {
- RpcConnectionPool pool = RpcConnectionPool.newPool(new TajoConf(), getClass().getSimpleName(), 2);
+ RpcConnectionPool pool = RpcConnectionPool.newPool(getClass().getSimpleName(), 2);
final SumRequest request = SumRequest.newBuilder()
.setX1(1)
.setX2(2)
@@ -187,7 +185,7 @@ public class TestBlockingRpc {
try {
int port = server.getListenAddress().getPort() + 1;
new BlockingRpcClient(DummyProtocol.class,
- NetUtils.getConnectAddress(new InetSocketAddress("127.0.0.1", port)), clientChannelFactory, retries);
+ RpcUtils.getConnectAddress(new InetSocketAddress("127.0.0.1", port)), clientChannelFactory, retries);
fail("Connection should be failed.");
} catch (ConnectException ce) {
expected = true;
@@ -260,9 +258,9 @@ public class TestBlockingRpc {
client.close();
client = null;
- String hostAndPort = NetUtils.normalizeInetSocketAddress(server.getListenAddress());
+ String hostAndPort = RpcUtils.normalizeInetSocketAddress(server.getListenAddress());
client = new BlockingRpcClient(DummyProtocol.class,
- NetUtils.createUnresolved(hostAndPort), clientChannelFactory, retries);
+ RpcUtils.createUnresolved(hostAndPort), clientChannelFactory, retries);
BlockingInterface stub = client.getStub();
EchoMessage message = EchoMessage.newBuilder()
[5/5] tajo git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/tajo into index_support
Posted by ji...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Conflicts:
tajo-docs/src/main/sphinx/index/how_to_use.rst
tajo-docs/src/main/sphinx/index_overview.rst
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/ad78d2f6
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/ad78d2f6
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/ad78d2f6
Branch: refs/heads/index_support
Commit: ad78d2f62c9990f012d75af8cb10a00b47b27312
Parents: 767b9a4 c45d0ef
Author: Jihoon Son <ji...@apache.org>
Authored: Sat Jan 10 20:26:31 2015 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Sat Jan 10 20:26:31 2015 +0900
----------------------------------------------------------------------
CHANGES | 9 +
.../tajo/catalog/AbstractCatalogClient.java | 11 +-
.../apache/tajo/client/SessionConnection.java | 2 +-
.../org/apache/tajo/master/QueryInProgress.java | 4 +-
.../apache/tajo/master/TajoContainerProxy.java | 14 +-
.../tajo/master/container/TajoContainerId.java | 3 +-
.../master/container/TajoContainerIdPBImpl.java | 100 +++++++++++
.../master/container/TajoConverterUtils.java | 176 -------------------
.../master/container/TajoRecordFactory.java | 31 ----
.../container/TajoRecordFactoryPBImpl.java | 104 -----------
.../container/TajoRecordFactoryProvider.java | 70 --------
.../tajo/master/container/TajoRecords.java | 39 ----
.../impl/pb/TajoContainerIdPBImpl.java | 100 -----------
.../apache/tajo/querymaster/QueryMaster.java | 2 +-
.../tajo/worker/ExecutionBlockContext.java | 2 +-
.../tajo/worker/TajoResourceAllocator.java | 6 +-
.../java/org/apache/tajo/worker/TaskRunner.java | 2 +-
.../tajo/worker/WorkerHeartbeatService.java | 2 +-
.../ConnectivityCheckerRuleForTajoWorker.java | 2 +-
.../src/main/resources/webapps/admin/index.jsp | 4 +-
.../resources/webapps/worker/querytasks.jsp | 10 +-
tajo-docs/src/main/sphinx/index/how_to_use.rst | 8 +-
tajo-docs/src/main/sphinx/index_overview.rst | 4 +-
tajo-rpc/pom.xml | 27 +--
.../org/apache/tajo/rpc/AsyncRpcClient.java | 3 +-
.../org/apache/tajo/rpc/BlockingRpcClient.java | 5 +-
.../org/apache/tajo/rpc/NettyClientBase.java | 3 +-
.../org/apache/tajo/rpc/NettyServerBase.java | 4 +-
.../org/apache/tajo/rpc/RpcChannelFactory.java | 18 +-
.../org/apache/tajo/rpc/RpcConnectionPool.java | 13 +-
.../main/java/org/apache/tajo/rpc/RpcUtils.java | 68 +++++++
.../org/apache/tajo/rpc/ServerCallable.java | 5 +-
.../java/org/apache/tajo/rpc/TestAsyncRpc.java | 7 +-
.../org/apache/tajo/rpc/TestBlockingRpc.java | 12 +-
34 files changed, 259 insertions(+), 611 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/ad78d2f6/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
----------------------------------------------------------------------
diff --cc tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
index dc12614,1a2fd44..53e5ab4
--- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
+++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java
@@@ -41,9 -35,7 +35,8 @@@ import org.apache.tajo.rpc.RpcConnectio
import org.apache.tajo.rpc.ServerCallable;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.NullProto;
- import org.apache.tajo.ha.HAServiceUtil;
import org.apache.tajo.util.ProtoUtil;
+import org.apache.tajo.util.TUtil;
import java.net.InetSocketAddress;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/tajo/blob/ad78d2f6/tajo-client/src/main/java/org/apache/tajo/client/SessionConnection.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/ad78d2f6/tajo-docs/src/main/sphinx/index/how_to_use.rst
----------------------------------------------------------------------
diff --cc tajo-docs/src/main/sphinx/index/how_to_use.rst
index 82a8bc9,776d205..a452289
--- a/tajo-docs/src/main/sphinx/index/how_to_use.rst
+++ b/tajo-docs/src/main/sphinx/index/how_to_use.rst
@@@ -66,4 -66,4 +66,4 @@@ If you don't want to use the index sca
.. note::
- If the index scan feature is enabled, Tajo currently always performs index scan regardless of its efficiency. You should set this option when the expected number of retrieved tuples is sufficiently small.
- Once the index scan feature is enabled, Tajo currently always performs the index scan regardless of its efficiency. You should set this option when the expected number of retrieved tuples is sufficiently small.
++ Once the index scan feature is enabled, Tajo currently always performs the index scan regardless of its efficiency. You should set this option when the expected number of retrieved tuples is sufficiently small.
[3/5] tajo git commit: TAJO-1296: Remove obsolete classes from
tajo.master.container package.
Posted by ji...@apache.org.
TAJO-1296: Remove obsolete classes from tajo.master.container package.
Closes #346
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/5d5b1520
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/5d5b1520
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/5d5b1520
Branch: refs/heads/index_support
Commit: 5d5b152008d25d2605008cd896871b69fd472833
Parents: 3308aab
Author: Hyunsik Choi <hy...@apache.org>
Authored: Sat Jan 10 16:35:26 2015 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Sat Jan 10 16:35:26 2015 +0900
----------------------------------------------------------------------
CHANGES | 3 +
.../tajo/master/container/TajoContainerId.java | 3 +-
.../master/container/TajoContainerIdPBImpl.java | 100 +++++++++++
.../master/container/TajoConverterUtils.java | 176 -------------------
.../master/container/TajoRecordFactory.java | 31 ----
.../container/TajoRecordFactoryPBImpl.java | 104 -----------
.../container/TajoRecordFactoryProvider.java | 70 --------
.../tajo/master/container/TajoRecords.java | 39 ----
.../impl/pb/TajoContainerIdPBImpl.java | 100 -----------
.../java/org/apache/tajo/worker/TaskRunner.java | 2 +-
10 files changed, 105 insertions(+), 523 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/5d5b1520/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 8f32d54..646bd11 100644
--- a/CHANGES
+++ b/CHANGES
@@ -282,6 +282,9 @@ Release 0.9.1 - unreleased
TASKS
+ TAJO-1296: Remove obsolete classes from tajo.master.container package.
+ (hyunsik)
+
TAJO-1294: Add index documents. (jihoon)
TAJO-1280: Update the roles of Hyoungjun and Jihun in web site.
http://git-wip-us.apache.org/repos/asf/tajo/blob/5d5b1520/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerId.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerId.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerId.java
index 0de5fe0..7bc27c6 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerId.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerId.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.util.Records;
/**
* This class is borrowed from the following source code :
@@ -43,7 +42,7 @@ public abstract class TajoContainerId implements Comparable<TajoContainerId>{
@Unstable
public static TajoContainerId newInstance(ApplicationAttemptId appAttemptId,
int containerId) {
- TajoContainerId id = TajoRecords.newRecord(TajoContainerId.class);
+ TajoContainerId id = new TajoContainerIdPBImpl();
id.setId(containerId);
id.setApplicationAttemptId(appAttemptId);
id.build();
http://git-wip-us.apache.org/repos/asf/tajo/blob/5d5b1520/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerIdPBImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerIdPBImpl.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerIdPBImpl.java
new file mode 100644
index 0000000..cf9e012
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerIdPBImpl.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.container;
+
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
+
+import com.google.common.base.Preconditions;
+import org.apache.tajo.ipc.ContainerProtocol;
+import org.apache.tajo.master.container.TajoContainerId;
+
+/**
+ * This class is borrowed from the following source code :
+ * ${hadoop-yarn-common}/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java
+ *
+ */
+@Private
+@Unstable
+public class TajoContainerIdPBImpl extends TajoContainerId {
+ ContainerProtocol.TajoContainerIdProto proto = null;
+ ContainerProtocol.TajoContainerIdProto.Builder builder = null;
+ private ApplicationAttemptId applicationAttemptId = null;
+
+ public TajoContainerIdPBImpl() {
+ builder = ContainerProtocol.TajoContainerIdProto.newBuilder();
+ }
+
+ public TajoContainerIdPBImpl(ContainerProtocol.TajoContainerIdProto proto) {
+ this.proto = proto;
+ this.applicationAttemptId = convertFromProtoFormat(proto.getAppAttemptId());
+ }
+
+ public ContainerProtocol.TajoContainerIdProto getProto() {
+ return proto;
+ }
+
+ @Override
+ public int getId() {
+ Preconditions.checkNotNull(proto);
+ return proto.getId();
+ }
+
+ @Override
+ protected void setId(int id) {
+ Preconditions.checkNotNull(builder);
+ builder.setId((id));
+ }
+
+
+ @Override
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return this.applicationAttemptId;
+ }
+
+ @Override
+ protected void setApplicationAttemptId(ApplicationAttemptId atId) {
+ if (atId != null) {
+ Preconditions.checkNotNull(builder);
+ builder.setAppAttemptId(convertToProtoFormat(atId));
+ }
+ this.applicationAttemptId = atId;
+ }
+
+ private ApplicationAttemptIdPBImpl convertFromProtoFormat(
+ ApplicationAttemptIdProto p) {
+ return new ApplicationAttemptIdPBImpl(p);
+ }
+
+ private ApplicationAttemptIdProto convertToProtoFormat(
+ ApplicationAttemptId t) {
+ return ((ApplicationAttemptIdPBImpl)t).getProto();
+ }
+
+ @Override
+ protected void build() {
+ proto = builder.build();
+ builder = null;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/tajo/blob/5d5b1520/tajo-core/src/main/java/org/apache/tajo/master/container/TajoConverterUtils.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoConverterUtils.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoConverterUtils.java
index a6db654..88c4823 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoConverterUtils.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoConverterUtils.java
@@ -53,86 +53,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@Private
public class TajoConverterUtils {
- public static final String APPLICATION_PREFIX = "application";
public static final String CONTAINER_PREFIX = "container";
- public static final String APPLICATION_ATTEMPT_PREFIX = "appattempt";
-
- /**
- * return a hadoop path from a given url
- *
- * @param url
- * url to convert
- * @return path from {@link URL}
- * @throws URISyntaxException
- */
- public static Path getPathFromYarnURL(URL url) throws URISyntaxException {
- String scheme = url.getScheme() == null ? "" : url.getScheme();
-
- String authority = "";
- if (url.getHost() != null) {
- authority = url.getHost();
- if (url.getUserInfo() != null) {
- authority = url.getUserInfo() + "@" + authority;
- }
- if (url.getPort() > 0) {
- authority += ":" + url.getPort();
- }
- }
-
- return new Path(
- (new URI(scheme, authority, url.getFile(), null, null)).normalize());
- }
-
- /**
- * change from CharSequence to string for map key and value
- * @param env map for converting
- * @return string,string map
- */
- public static Map<String, String> convertToString(
- Map<CharSequence, CharSequence> env) {
-
- Map<String, String> stringMap = new HashMap<String, String>();
- for (Entry<CharSequence, CharSequence> entry: env.entrySet()) {
- stringMap.put(entry.getKey().toString(), entry.getValue().toString());
- }
- return stringMap;
- }
-
- public static URL getYarnUrlFromPath(Path path) {
- return getYarnUrlFromURI(path.toUri());
- }
-
- public static URL getYarnUrlFromURI(URI uri) {
- URL url = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(URL.class);
- if (uri.getHost() != null) {
- url.setHost(uri.getHost());
- }
- if (uri.getUserInfo() != null) {
- url.setUserInfo(uri.getUserInfo());
- }
- url.setPort(uri.getPort());
- url.setScheme(uri.getScheme());
- url.setFile(uri.getPath());
- return url;
- }
-
- public static String toString(ApplicationId appId) {
- return appId.toString();
- }
-
- public static ApplicationId toApplicationId(RecordFactory recordFactory,
- String appIdStr) {
- Iterator<String> it = _split(appIdStr).iterator();
- it.next(); // prefix. TODO: Validate application prefix
- return toApplicationId(recordFactory, it);
- }
-
- private static ApplicationId toApplicationId(RecordFactory recordFactory,
- Iterator<String> it) {
- ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
- Integer.parseInt(it.next()));
- return appId;
- }
private static ApplicationAttemptId toApplicationAttemptId(
Iterator<String> it) throws NumberFormatException {
@@ -143,32 +64,10 @@ public class TajoConverterUtils {
return appAttemptId;
}
- private static ApplicationId toApplicationId(
- Iterator<String> it) throws NumberFormatException {
- ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
- Integer.parseInt(it.next()));
- return appId;
- }
-
public static String toString(TajoContainerId cId) {
return cId == null ? null : cId.toString();
}
- public static NodeId toNodeId(String nodeIdStr) {
- String[] parts = nodeIdStr.split(":");
- if (parts.length != 2) {
- throw new IllegalArgumentException("Invalid NodeId [" + nodeIdStr
- + "]. Expected host:port");
- }
- try {
- NodeId nodeId =
- NodeId.newInstance(parts[0], Integer.parseInt(parts[1]));
- return nodeId;
- } catch (NumberFormatException e) {
- throw new IllegalArgumentException("Invalid port: " + parts[1], e);
- }
- }
-
public static TajoContainerId toTajoContainerId(String containerIdStr) {
Iterator<String> it = _split(containerIdStr).iterator();
if (!it.next().equals(CONTAINER_PREFIX)) {
@@ -185,79 +84,4 @@ public class TajoConverterUtils {
+ containerIdStr, n);
}
}
-
- public static ApplicationAttemptId toApplicationAttemptId(
- String applicationAttmeptIdStr) {
- Iterator<String> it = _split(applicationAttmeptIdStr).iterator();
- if (!it.next().equals(APPLICATION_ATTEMPT_PREFIX)) {
- throw new IllegalArgumentException("Invalid AppAttemptId prefix: "
- + applicationAttmeptIdStr);
- }
- try {
- return toApplicationAttemptId(it);
- } catch (NumberFormatException n) {
- throw new IllegalArgumentException("Invalid AppAttemptId: "
- + applicationAttmeptIdStr, n);
- }
- }
-
- public static ApplicationId toApplicationId(
- String appIdStr) {
- Iterator<String> it = _split(appIdStr).iterator();
- if (!it.next().equals(APPLICATION_PREFIX)) {
- throw new IllegalArgumentException("Invalid ApplicationId prefix: "
- + appIdStr + ". The valid ApplicationId should start with prefix "
- + APPLICATION_PREFIX);
- }
- try {
- return toApplicationId(it);
- } catch (NumberFormatException n) {
- throw new IllegalArgumentException("Invalid AppAttemptId: "
- + appIdStr, n);
- }
- }
-
- /**
- * Convert a protobuf token into a rpc token and set its service. Supposed
- * to be used for tokens other than RMDelegationToken. For
- * RMDelegationToken, use
- * {@link #convertFromYarn(org.apache.hadoop.yarn.api.records.Token,
- * org.apache.hadoop.io.Text)} instead.
- *
- * @param protoToken the yarn token
- * @param serviceAddr the connect address for the service
- * @return rpc token
- */
- public static <T extends TokenIdentifier> Token<T> convertFromYarn(
- org.apache.hadoop.yarn.api.records.Token protoToken,
- InetSocketAddress serviceAddr) {
- Token<T> token = new Token<T>(protoToken.getIdentifier().array(),
- protoToken.getPassword().array(),
- new Text(protoToken.getKind()),
- new Text(protoToken.getService()));
- if (serviceAddr != null) {
- SecurityUtil.setTokenService(token, serviceAddr);
- }
- return token;
- }
-
- /**
- * Convert a protobuf token into a rpc token and set its service.
- *
- * @param protoToken the yarn token
- * @param service the service for the token
- */
- public static <T extends TokenIdentifier> Token<T> convertFromYarn(
- org.apache.hadoop.yarn.api.records.Token protoToken,
- Text service) {
- Token<T> token = new Token<T>(protoToken.getIdentifier().array(),
- protoToken.getPassword().array(),
- new Text(protoToken.getKind()),
- new Text(protoToken.getService()));
-
- if (service != null) {
- token.setService(service);
- }
- return token;
- }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5d5b1520/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactory.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactory.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactory.java
deleted file mode 100644
index 2fd8697..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactory.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.container;
-
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-
-/**
- * This class is borrowed from the following source code :
- * ${hadoop-yarn-api}/src/main/java/org/apache/hadoop/yarn/factories/RecordFactory.java
- *
- */
-@Unstable
-public interface TajoRecordFactory {
- public <T> T newRecordInstance(Class<T> clazz);
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5d5b1520/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryPBImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryPBImpl.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryPBImpl.java
deleted file mode 100644
index c352a28..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryPBImpl.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.tajo.master.container;
-
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.tajo.master.container.TajoRecordFactory;
-
-/**
- * This class is borrowed from the following source code :
- * ${hadoop-yarn-common}/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/RecordFactoryPBImpl.java
- */
-@Private
-public class TajoRecordFactoryPBImpl implements TajoRecordFactory {
-
- private static final String PB_IMPL_PACKAGE_SUFFIX = "impl.pb";
- private static final String PB_IMPL_CLASS_SUFFIX = "PBImpl";
-
- private static final TajoRecordFactoryPBImpl self = new TajoRecordFactoryPBImpl();
- private Configuration localConf = new Configuration();
- private ConcurrentMap<Class<?>, Constructor<?>> cache = new ConcurrentHashMap<Class<?>, Constructor<?>>();
-
- private TajoRecordFactoryPBImpl() {
- }
-
- public static TajoRecordFactory get() {
- return self;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <T> T newRecordInstance(Class<T> clazz) {
-
- Constructor<?> constructor = cache.get(clazz);
- if (constructor == null) {
- Class<?> pbClazz = null;
- try {
- pbClazz = localConf.getClassByName(getPBImplClassName(clazz));
- } catch (ClassNotFoundException e) {
- throw new YarnRuntimeException("Failed to load class: ["
- + getPBImplClassName(clazz) + "]", e);
- }
- try {
- constructor = pbClazz.getConstructor();
- constructor.setAccessible(true);
- cache.putIfAbsent(clazz, constructor);
- } catch (NoSuchMethodException e) {
- throw new YarnRuntimeException("Could not find 0 argument constructor", e);
- }
- }
- try {
- Object retObject = constructor.newInstance();
- return (T)retObject;
- } catch (InvocationTargetException e) {
- throw new YarnRuntimeException(e);
- } catch (IllegalAccessException e) {
- throw new YarnRuntimeException(e);
- } catch (InstantiationException e) {
- throw new YarnRuntimeException(e);
- }
- }
-
- private String getPBImplClassName(Class<?> clazz) {
- String srcPackagePart = getPackageName(clazz);
- String srcClassName = getClassName(clazz);
- String destPackagePart = srcPackagePart + "." + PB_IMPL_PACKAGE_SUFFIX;
- String destClassPart = srcClassName + PB_IMPL_CLASS_SUFFIX;
- return destPackagePart + "." + destClassPart;
- }
-
- private String getClassName(Class<?> clazz) {
- String fqName = clazz.getName();
- return (fqName.substring(fqName.lastIndexOf(".") + 1, fqName.length()));
- }
-
- private String getPackageName(Class<?> clazz) {
- return clazz.getPackage().getName();
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5d5b1520/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryProvider.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryProvider.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryProvider.java
deleted file mode 100644
index c260e85..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryProvider.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.container;
-
-
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-
-/**
- * This class is borrowed from the following source code :
- * ${hadoop-yarn-api}/src/main/java/org/apache/hadoop/yarn/factory/providers/RecordFactoryProvider.java
- */
-@Unstable
-public class TajoRecordFactoryProvider {
- private static Configuration defaultConf;
-
- static {
- defaultConf = new Configuration();
- }
-
- private TajoRecordFactoryProvider() {
- }
-
- public static TajoRecordFactory getRecordFactory(Configuration conf) {
- if (conf == null) {
- //Assuming the default configuration has the correct factories set.
- //Users can specify a particular factory by providing a configuration.
- conf = defaultConf;
- }
- return (TajoRecordFactory) getFactoryClassInstance(TajoRecordFactoryPBImpl.class.getCanonicalName());
- }
-
- private static Object getFactoryClassInstance(String factoryClassName) {
- try {
- Class<?> clazz = Class.forName(factoryClassName);
- Method method = clazz.getMethod("get", null);
- method.setAccessible(true);
- return method.invoke(null, null);
- } catch (ClassNotFoundException e) {
- throw new YarnRuntimeException(e);
- } catch (NoSuchMethodException e) {
- throw new YarnRuntimeException(e);
- } catch (InvocationTargetException e) {
- throw new YarnRuntimeException(e);
- } catch (IllegalAccessException e) {
- throw new YarnRuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5d5b1520/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecords.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecords.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecords.java
deleted file mode 100644
index e85edf9..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecords.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.container;
-
-
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-
-/**
- * This class is borrowed from the following source code :
- * ${hadoop-yarn-api}/src/main/java/org/apache/hadoop/yarn/util/Records.java
- *
- * Convenient API record utils
- */
-@Unstable
-public class TajoRecords {
- // The default record factory
- private static final TajoRecordFactory factory =
- TajoRecordFactoryProvider.getRecordFactory(null);
-
- public static <T> T newRecord(Class<T> cls) {
- return factory.newRecordInstance(cls);
- }
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/5d5b1520/tajo-core/src/main/java/org/apache/tajo/master/container/impl/pb/TajoContainerIdPBImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/impl/pb/TajoContainerIdPBImpl.java b/tajo-core/src/main/java/org/apache/tajo/master/container/impl/pb/TajoContainerIdPBImpl.java
deleted file mode 100644
index 9d31050..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/container/impl/pb/TajoContainerIdPBImpl.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.container.impl.pb;
-
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
-import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
-
-import com.google.common.base.Preconditions;
-import org.apache.tajo.ipc.ContainerProtocol;
-import org.apache.tajo.master.container.TajoContainerId;
-
-/**
- * This class is borrowed from the following source code :
- * ${hadoop-yarn-common}/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java
- *
- */
-@Private
-@Unstable
-public class TajoContainerIdPBImpl extends TajoContainerId {
- ContainerProtocol.TajoContainerIdProto proto = null;
- ContainerProtocol.TajoContainerIdProto.Builder builder = null;
- private ApplicationAttemptId applicationAttemptId = null;
-
- public TajoContainerIdPBImpl() {
- builder = ContainerProtocol.TajoContainerIdProto.newBuilder();
- }
-
- public TajoContainerIdPBImpl(ContainerProtocol.TajoContainerIdProto proto) {
- this.proto = proto;
- this.applicationAttemptId = convertFromProtoFormat(proto.getAppAttemptId());
- }
-
- public ContainerProtocol.TajoContainerIdProto getProto() {
- return proto;
- }
-
- @Override
- public int getId() {
- Preconditions.checkNotNull(proto);
- return proto.getId();
- }
-
- @Override
- protected void setId(int id) {
- Preconditions.checkNotNull(builder);
- builder.setId((id));
- }
-
-
- @Override
- public ApplicationAttemptId getApplicationAttemptId() {
- return this.applicationAttemptId;
- }
-
- @Override
- protected void setApplicationAttemptId(ApplicationAttemptId atId) {
- if (atId != null) {
- Preconditions.checkNotNull(builder);
- builder.setAppAttemptId(convertToProtoFormat(atId));
- }
- this.applicationAttemptId = atId;
- }
-
- private ApplicationAttemptIdPBImpl convertFromProtoFormat(
- ApplicationAttemptIdProto p) {
- return new ApplicationAttemptIdPBImpl(p);
- }
-
- private ApplicationAttemptIdProto convertToProtoFormat(
- ApplicationAttemptId t) {
- return ((ApplicationAttemptIdPBImpl)t).getProto();
- }
-
- @Override
- protected void build() {
- proto = builder.build();
- builder = null;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/tajo/blob/5d5b1520/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
index f0da0cd..23efffa 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
@@ -31,7 +31,7 @@ import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.engine.query.TaskRequestImpl;
import org.apache.tajo.ipc.QueryMasterProtocol.QueryMasterProtocolService;
import org.apache.tajo.master.container.TajoContainerId;
-import org.apache.tajo.master.container.impl.pb.TajoContainerIdPBImpl;
+import org.apache.tajo.master.container.TajoContainerIdPBImpl;
import org.apache.tajo.master.container.TajoConverterUtils;
import org.apache.tajo.rpc.CallFuture;
import org.apache.tajo.rpc.NullCallback;
[4/5] tajo git commit: TAJO-1297: Tajo Web UI does not work after
TAJO-1291. (jihoon)
Posted by ji...@apache.org.
TAJO-1297: Tajo Web UI does not work after TAJO-1291. (jihoon)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/c45d0ef0
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/c45d0ef0
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/c45d0ef0
Branch: refs/heads/index_support
Commit: c45d0ef02df1aa59bb54e0da0205890cc050fcff
Parents: 5d5b152
Author: Jihoon Son <ji...@apache.org>
Authored: Sat Jan 10 17:04:44 2015 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Sat Jan 10 17:04:44 2015 +0900
----------------------------------------------------------------------
CHANGES | 2 ++
tajo-core/src/main/resources/webapps/admin/index.jsp | 4 ++--
.../src/main/resources/webapps/worker/querytasks.jsp | 10 +++++-----
3 files changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/c45d0ef0/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 646bd11..5f427a9 100644
--- a/CHANGES
+++ b/CHANGES
@@ -161,6 +161,8 @@ Release 0.9.1 - unreleased
BUG FIXES
+ TAJO-1297: Tajo Web UI does not work after TAJO-1291. (jihoon)
+
TAJO-1251: Query is hanging occasionally by shuffle report. (jinho)
TAJO-1287: Repeated using of the same order by key in multiple
http://git-wip-us.apache.org/repos/asf/tajo/blob/c45d0ef0/tajo-core/src/main/resources/webapps/admin/index.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/admin/index.jsp b/tajo-core/src/main/resources/webapps/admin/index.jsp
index 0defb3c..d98abfd 100644
--- a/tajo-core/src/main/resources/webapps/admin/index.jsp
+++ b/tajo-core/src/main/resources/webapps/admin/index.jsp
@@ -21,7 +21,7 @@
<%@ page import="org.apache.hadoop.fs.FileSystem" %>
<%@ page import="org.apache.tajo.conf.TajoConf" %>
-<%@ page import="org.apache.tajo.ipc.TajoMasterProtocol" %>
+<%@ page import="org.apache.tajo.ipc.QueryCoordinatorProtocol" %>
<%@ page import="org.apache.tajo.master.TajoMaster" %>
<%@ page import="org.apache.tajo.ha.HAService" %>
<%@ page import="org.apache.tajo.ha.TajoMasterInfo" %>
@@ -52,7 +52,7 @@
int runningQueryMasterTask = 0;
- TajoMasterProtocol.ClusterResourceSummary clusterResourceSummary =
+ QueryCoordinatorProtocol.ClusterResourceSummary clusterResourceSummary =
master.getContext().getResourceManager().getClusterResourceSummary();
for(Worker eachWorker: workers.values()) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/c45d0ef0/tajo-core/src/main/resources/webapps/worker/querytasks.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/worker/querytasks.jsp b/tajo-core/src/main/resources/webapps/worker/querytasks.jsp
index 6d0e3a2..3e3c2c2 100644
--- a/tajo-core/src/main/resources/webapps/worker/querytasks.jsp
+++ b/tajo-core/src/main/resources/webapps/worker/querytasks.jsp
@@ -24,7 +24,7 @@
<%@ page import="org.apache.tajo.TaskAttemptId" %>
<%@ page import="org.apache.tajo.catalog.statistics.TableStats" %>
<%@ page import="org.apache.tajo.plan.util.PlannerUtil" %>
-<%@ page import="org.apache.tajo.ipc.TajoMasterProtocol" %>
+<%@ page import="org.apache.tajo.ipc.QueryCoordinatorProtocol" %>
<%@ page import="org.apache.tajo.querymaster.*" %>
<%@ page import="org.apache.tajo.webapp.StaticHttpServer" %>
<%@ page import="org.apache.tajo.worker.TajoWorker" %>
@@ -60,12 +60,12 @@
}
TajoWorker tajoWorker = (TajoWorker) StaticHttpServer.getInstance().getAttribute("tajo.info.server.object");
- List<TajoMasterProtocol.WorkerResourceProto> allWorkers = tajoWorker.getWorkerContext()
+ List<QueryCoordinatorProtocol.WorkerResourceProto> allWorkers = tajoWorker.getWorkerContext()
.getQueryMasterManagerService().getQueryMaster().getAllWorker();
- Map<Integer, TajoMasterProtocol.WorkerResourceProto> workerMap = new HashMap<Integer, TajoMasterProtocol.WorkerResourceProto>();
+ Map<Integer, QueryCoordinatorProtocol.WorkerResourceProto> workerMap = new HashMap<Integer, QueryCoordinatorProtocol.WorkerResourceProto>();
if(allWorkers != null) {
- for(TajoMasterProtocol.WorkerResourceProto eachWorker: allWorkers) {
+ for(QueryCoordinatorProtocol.WorkerResourceProto eachWorker: allWorkers) {
workerMap.put(eachWorker.getConnectionInfo().getId(), eachWorker);
}
}
@@ -230,7 +230,7 @@
String taskHost = eachTask.getSucceededHost() == null ? "-" : eachTask.getSucceededHost();
if(eachTask.getSucceededHost() != null) {
- TajoMasterProtocol.WorkerResourceProto worker =
+ QueryCoordinatorProtocol.WorkerResourceProto worker =
workerMap.get(eachTask.getLastAttempt().getWorkerConnectionInfo().getId());
if(worker != null) {
TaskAttempt lastAttempt = eachTask.getLastAttempt();
[2/5] tajo git commit: TAJO-1294: Add index documents. (jihoon)
Posted by ji...@apache.org.
TAJO-1294: Add index documents. (jihoon)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/3308aab4
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/3308aab4
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/3308aab4
Branch: refs/heads/index_support
Commit: 3308aab4b64d36b805b0b0e6cc282a6dce97f6ad
Parents: 9eac34f
Author: Jihoon Son <ji...@apache.org>
Authored: Sat Jan 10 16:27:23 2015 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Sat Jan 10 16:27:23 2015 +0900
----------------------------------------------------------------------
CHANGES | 2 +
tajo-docs/src/main/sphinx/index.rst | 1 +
tajo-docs/src/main/sphinx/index/future_work.rst | 8 +++
tajo-docs/src/main/sphinx/index/how_to_use.rst | 69 ++++++++++++++++++++
tajo-docs/src/main/sphinx/index/types.rst | 7 ++
tajo-docs/src/main/sphinx/index_overview.rst | 20 ++++++
tajo-docs/src/main/sphinx/sql_language/ddl.rst | 33 +++++++++-
7 files changed, 139 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/3308aab4/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index b5578e7..8f32d54 100644
--- a/CHANGES
+++ b/CHANGES
@@ -282,6 +282,8 @@ Release 0.9.1 - unreleased
TASKS
+ TAJO-1294: Add index documents. (jihoon)
+
TAJO-1280: Update the roles of Hyoungjun and Jihun in web site.
(hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/3308aab4/tajo-docs/src/main/sphinx/index.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/index.rst b/tajo-docs/src/main/sphinx/index.rst
index 80cd842..667f270 100644
--- a/tajo-docs/src/main/sphinx/index.rst
+++ b/tajo-docs/src/main/sphinx/index.rst
@@ -37,6 +37,7 @@ Table of Contents:
functions
table_management
table_partitioning
+ index_overview
backup_and_restore
hcatalog_integration
jdbc_driver
http://git-wip-us.apache.org/repos/asf/tajo/blob/3308aab4/tajo-docs/src/main/sphinx/index/future_work.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/index/future_work.rst b/tajo-docs/src/main/sphinx/index/future_work.rst
new file mode 100644
index 0000000..c6ec47d
--- /dev/null
+++ b/tajo-docs/src/main/sphinx/index/future_work.rst
@@ -0,0 +1,8 @@
+*************************************
+Future Works
+*************************************
+
+* Providing more index types, such as bitmap and HBase index
+* Supporting index on partitioned tables
+* Supporting the backup and restore feature
+* Cost-based query optimization by estimating the query selectivity
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/3308aab4/tajo-docs/src/main/sphinx/index/how_to_use.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/index/how_to_use.rst b/tajo-docs/src/main/sphinx/index/how_to_use.rst
new file mode 100644
index 0000000..776d205
--- /dev/null
+++ b/tajo-docs/src/main/sphinx/index/how_to_use.rst
@@ -0,0 +1,69 @@
+*************************************
+How to use index?
+*************************************
+
+-------------------------------------
+1. Create index
+-------------------------------------
+
+The first step for utilizing index is index creation. You can create index using SQL (:doc:`/sql_language/ddl`) or Tajo API (:doc:`/tajo_client_api`). For example, you can create a BST index on the lineitem table by submitting the following SQL to Tajo.
+
+.. code-block:: sql
+
+ default> create index l_orderkey_idx on lineitem (l_orderkey);
+
+If the index is created successfully, you can see the information about that index as follows: ::
+
+ default> \d lineitem
+
+ table name: default.lineitem
+ table path: hdfs://localhost:7020/tpch/lineitem
+ store type: CSV
+ number of rows: unknown
+ volume: 753.9 MB
+ Options:
+ 'text.delimiter'='|'
+
+ schema:
+ l_orderkey INT8
+ l_partkey INT8
+ l_suppkey INT8
+ l_linenumber INT8
+ l_quantity FLOAT4
+ l_extendedprice FLOAT4
+ l_discount FLOAT4
+ l_tax FLOAT4
+ l_returnflag TEXT
+ l_linestatus TEXT
+ l_shipdate DATE
+ l_commitdate DATE
+ l_receiptdate DATE
+ l_shipinstruct TEXT
+ l_shipmode TEXT
+ l_comment TEXT
+
+
+ Indexes:
+ "l_orderkey_idx" TWO_LEVEL_BIN_TREE (l_orderkey ASC NULLS LAST )
+
+For more information about index creation, please refer to the above links.
+
+-------------------------------------
+2. Enable/disable index scans
+-------------------------------------
+
+When an index is successfully created, you must enable the index scan feature as follows:
+
+.. code-block:: sql
+
+ default> \set INDEX_ENABLED true
+
+If you don't want to use the index scan feature anymore, you can simply disable it as follows:
+
+.. code-block:: sql
+
+ default> \set INDEX_ENABLED false
+
+.. note::
+
+ Once the index scan feature is enabled, Tajo currently always performs the index scan regardless of its efficiency. You should set this option when the expected number of retrieved tuples is sufficiently small.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/3308aab4/tajo-docs/src/main/sphinx/index/types.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/index/types.rst b/tajo-docs/src/main/sphinx/index/types.rst
new file mode 100644
index 0000000..457f453
--- /dev/null
+++ b/tajo-docs/src/main/sphinx/index/types.rst
@@ -0,0 +1,7 @@
+*************************************
+Index Types
+*************************************
+
+Currently, Tajo supports only one type of index, ``TWO_LEVEL_BIN_TREE``, shortly ``BST``. The BST index is a kind of binary search tree which is extended to be permanently stored on disk. It consists of two levels of nodes; a leaf node indexes the keys with the positions of data in an HDFS block and a root node indexes the keys with the leaf node indices.
+
+When an index scan is started, the query engine first reads the root node and finds the search key. If it finds a leaf node corresponding to the search key, it subsequently finds the search key in that leaf node. Finally, it directly reads a tuple corresponding to the search key from HDFS.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/3308aab4/tajo-docs/src/main/sphinx/index_overview.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/index_overview.rst b/tajo-docs/src/main/sphinx/index_overview.rst
new file mode 100644
index 0000000..a42931b
--- /dev/null
+++ b/tajo-docs/src/main/sphinx/index_overview.rst
@@ -0,0 +1,20 @@
+*****************************
+Index (Experimental Feature)
+*****************************
+
+An index is a data structure that is used for efficient query processing. Using an index, the Tajo query engine can directly retrieve search values.
+
+This is still an experimental feature. In order to use indexes, you must check out the source code of the ``index_support`` branch::
+
+ git clone -b index_support https://git-wip-us.apache.org/repos/asf/tajo.git tajo-index
+
+For the source code build, please refer to :doc:`getting_started`.
+
+The following sections describe the supported index types, the query execution with an index, and the future works.
+
+.. toctree::
+ :maxdepth: 1
+
+ index/types
+ index/how_to_use
+ index/future_work
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/3308aab4/tajo-docs/src/main/sphinx/sql_language/ddl.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/sql_language/ddl.rst b/tajo-docs/src/main/sphinx/sql_language/ddl.rst
index 3fba6be..60b7190 100644
--- a/tajo-docs/src/main/sphinx/sql_language/ddl.rst
+++ b/tajo-docs/src/main/sphinx/sql_language/ddl.rst
@@ -75,4 +75,35 @@ If you want to add an external table that contains compressed data, you should g
DROP TABLE [IF EXISTS] <table_name> [PURGE]
-``IF EXISTS`` allows ``DROP DATABASE`` statement to avoid an error which occurs when the database does not exist. ``DROP TABLE`` statement removes a table from Tajo catalog, but it does not remove the contents. If ``PURGE`` option is given, ``DROP TABLE`` statement will eliminate the entry in the catalog as well as the contents.
\ No newline at end of file
+``IF EXISTS`` allows ``DROP DATABASE`` statement to avoid an error which occurs when the database does not exist. ``DROP TABLE`` statement removes a table from Tajo catalog, but it does not remove the contents. If ``PURGE`` option is given, ``DROP TABLE`` statement will eliminate the entry in the catalog as well as the contents.
+
+========================
+ CREATE INDEX
+========================
+
+*Synopsis*
+
+.. code-block:: sql
+
+ CREATE INDEX [ name ] ON table_name [ USING method ]
+ ( { column_name | ( expression ) } [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [, ...] )
+ [ WHERE predicate ]
+
+------------------------
+ Index method
+------------------------
+
+Currently, Tajo supports only one type of index.
+
+Index methods:
+ * TWO_LEVEL_BIN_TREE: This method is used by default in Tajo. For more information about its structure, please refer to :doc:`/index/types`.
+
+========================
+ DROP INDEX
+========================
+
+*Synopsis*
+
+.. code-block:: sql
+
+ DROP INDEX name
\ No newline at end of file