You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2013/08/30 08:57:20 UTC
git commit: TAJO-141: Set on demand as the default cluster mode.
(hyunsik)
Updated Branches:
refs/heads/master ce84eba43 -> 466adcc57
TAJO-141: Set on demand as the default cluster mode. (hyunsik)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/466adcc5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/466adcc5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/466adcc5
Branch: refs/heads/master
Commit: 466adcc57f19005c6f3a9bbbc5cbf09a3d77fdc4
Parents: ce84eba
Author: Hyunsik Choi <hy...@apache.org>
Authored: Fri Aug 30 15:55:12 2013 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Fri Aug 30 15:55:12 2013 +0900
----------------------------------------------------------------------
.../main/java/org/apache/tajo/conf/TajoConf.java | 2 +-
.../tajo/master/TajoMasterClientService.java | 2 +-
.../org/apache/tajo/master/TajoMasterService.java | 3 +--
.../apache/tajo/master/querymaster/SubQuery.java | 2 +-
.../tajo/master/rm/YarnTajoResourceManager.java | 2 +-
.../src/main/resources/tajo-default.xml | 17 ++---------------
.../java/org/apache/tajo/TajoTestingCluster.java | 12 ++++++++++++
.../src/test/resources/tajo-default.xml | 7 ++++++-
tajo-dist/src/main/conf/tajo-env.sh | 3 ++-
.../org/apache/tajo/rpc/TestProtoAsyncRpc.java | 4 +++-
10 files changed, 30 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/466adcc5/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
index 2fe64e6..9bdda7e 100644
--- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
+++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
@@ -66,7 +66,7 @@ public class TajoConf extends YarnConfiguration {
// Service Addresses
TASKRUNNER_LISTENER_ADDRESS("tajo.master.taskrunnerlistener.addr", "0.0.0.0:0"), // used internally
CLIENT_SERVICE_ADDRESS("tajo.master.clientservice.addr", "127.0.0.1:9004"),
- TAJO_MASTER_SERVICE_ADDRESS("tajo.master.manager.addr", "127.0.0.1:9005"),
+ TAJO_MASTER_SERVICE_ADDRESS("tajo.master.manager.addr", "0.0.0.0:9005"),
//////////////////////////////////
// Catalog Configuration
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/466adcc5/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
index 03426bf..e65c1f3 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
@@ -90,7 +90,7 @@ public class TajoMasterClientService extends AbstractService {
LOG.error(e);
}
server.start();
- bindAddress = server.getListenAddress();
+ bindAddress = NetUtils.getConnectAddress(server.getListenAddress());
this.conf.setVar(ConfVars.CLIENT_SERVICE_ADDRESS, NetUtils.normalizeInetSocketAddress(bindAddress));
LOG.info("Instantiated TajoMasterClientService at " + this.bindAddress);
super.start();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/466adcc5/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterService.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterService.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterService.java
index 8d1bbe0..d518dce 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterService.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterService.java
@@ -58,7 +58,6 @@ public class TajoMasterService extends AbstractService {
@Override
public void start() {
- // TODO resolve hostname
String confMasterServiceAddr = conf.getVar(TajoConf.ConfVars.TAJO_MASTER_SERVICE_ADDRESS);
InetSocketAddress initIsa = NetUtils.createSocketAddr(confMasterServiceAddr);
try {
@@ -67,7 +66,7 @@ public class TajoMasterService extends AbstractService {
LOG.error(e);
}
server.start();
- bindAddress = server.getListenAddress();
+ bindAddress = NetUtils.getConnectAddress(server.getListenAddress());
this.conf.setVar(TajoConf.ConfVars.TAJO_MASTER_SERVICE_ADDRESS,
NetUtils.normalizeInetSocketAddress(bindAddress));
LOG.info("Instantiated TajoMasterService at " + this.bindAddress);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/466adcc5/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
index ce4c846..89d3fed 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
@@ -626,7 +626,7 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
final Resource resource = Records.newRecord(Resource.class);
- resource.setMemory(512);
+ resource.setMemory(2048);
LOG.info("Request Container for " + subQuery.getId() + " containers=" + numRequest);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/466adcc5/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/rm/YarnTajoResourceManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/rm/YarnTajoResourceManager.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/rm/YarnTajoResourceManager.java
index 9bb153c..fe31bc6 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/rm/YarnTajoResourceManager.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/rm/YarnTajoResourceManager.java
@@ -268,7 +268,7 @@ public class YarnTajoResourceManager implements WorkerResourceManager {
final Resource resource = Records.newRecord(Resource.class);
// TODO - get default value from conf
- resource.setMemory(256);
+ resource.setMemory(2048);
resource.setVirtualCores(1);
Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/466adcc5/tajo-core/tajo-core-backend/src/main/resources/tajo-default.xml
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/resources/tajo-default.xml b/tajo-core/tajo-core-backend/src/main/resources/tajo-default.xml
index e8ad503..2025110 100644
--- a/tajo-core/tajo-core-backend/src/main/resources/tajo-default.xml
+++ b/tajo-core/tajo-core-backend/src/main/resources/tajo-default.xml
@@ -42,19 +42,6 @@
<description>ms</description>
</property>
-
-
- <property>
- <name>tajo.master.clientservice.addr</name>
- <value>127.0.0.1:9004</value>
- </property>
-
- <property>
- <name>tajo.master.manager.addr</name>
- <value>127.0.0.1:9005</value>
- <description>rpc port for tajo worker</description>
- </property>
-
<property>
<name>tajo.query.session.timeout</name>
<value>60000</value>
@@ -63,7 +50,7 @@
<property>
<name>tajo.resource.manager</name>
- <value>org.apache.tajo.master.rm.TajoWorkerResourceManager</value>
+ <value>org.apache.tajo.master.rm.YarnTajoResourceManager</value>
<description>This can be org.apache.tajo.master.rm.TajoWorkerResourceManager or org.apache.tajo.master.rm.YarnTajoResourceManager</description>
</property>
@@ -111,4 +98,4 @@
<value>4</value>
<description>The number of CPU cores on a worker</description>
</property>
-</configuration>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/466adcc5/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java
index 2588e46..f031938 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java
@@ -19,6 +19,7 @@
package org.apache.tajo;
import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
import com.google.common.io.Closeables;
import com.google.common.io.Files;
import org.apache.commons.logging.Log;
@@ -37,6 +38,7 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.master.TajoMaster;
import org.apache.tajo.master.rm.TajoWorkerResourceManager;
+import org.apache.tajo.master.rm.YarnTajoResourceManager;
import org.apache.tajo.util.NetUtils;
import org.apache.tajo.worker.TajoWorker;
@@ -79,6 +81,16 @@ public class TajoTestingCluster {
public TajoTestingCluster() {
this.conf = new TajoConf();
+ if (System.getProperty("tajo.resource.manager") != null) {
+ String testResourceManager = System.getProperty("tajo.resource.manager");
+ Preconditions.checkState(
+ testResourceManager.equals(TajoWorkerResourceManager.class.getCanonicalName()) ||
+ testResourceManager.equals(YarnTajoResourceManager.class.getCanonicalName()),
+ "tajo.resource.manager must be either " + TajoWorkerResourceManager.class.getCanonicalName() + " or " +
+ YarnTajoResourceManager.class.getCanonicalName() +"."
+ );
+ this.conf.set("tajo.resource.manager", System.getProperty("tajo.resource.manager"));
+ }
this.standbyWorkerMode =
this.conf.get("tajo.resource.manager", TajoWorkerResourceManager.class.getCanonicalName())
.indexOf(TajoWorkerResourceManager.class.getName()) >= 0;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/466adcc5/tajo-core/tajo-core-backend/src/test/resources/tajo-default.xml
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/resources/tajo-default.xml b/tajo-core/tajo-core-backend/src/test/resources/tajo-default.xml
index 20b80aa..aedaac3 100644
--- a/tajo-core/tajo-core-backend/src/test/resources/tajo-default.xml
+++ b/tajo-core/tajo-core-backend/src/test/resources/tajo-default.xml
@@ -39,6 +39,11 @@
<property>
<name>tajo.resource.manager</name>
<value>org.apache.tajo.master.rm.TajoWorkerResourceManager</value>
- <description>org.apache.tajo.master.rm.TajoWorkerResourceManager or org.apache.tajo.master.rm.YarnTajoResourceManager</description>
</property>
+
+ <property>
+ <name>tajo.master.manager.addr</name>
+ <value>127.0.0.1</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/466adcc5/tajo-dist/src/main/conf/tajo-env.sh
----------------------------------------------------------------------
diff --git a/tajo-dist/src/main/conf/tajo-env.sh b/tajo-dist/src/main/conf/tajo-env.sh
index 2abc448..b70fe01 100755
--- a/tajo-dist/src/main/conf/tajo-env.sh
+++ b/tajo-dist/src/main/conf/tajo-env.sh
@@ -53,5 +53,6 @@ export TAJO_OPTS=-XX:+PrintGCTimeStamps
# export TAJO_NICENESS=10
# Tajo Distributed Execution Mode
-export TAJO_WORKER_STANDBY_MODE=true
+# the default mode is on-demand mode using YarnTajoResourceManager.
+# export TAJO_WORKER_STANDBY_MODE=yes
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/466adcc5/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestProtoAsyncRpc.java
----------------------------------------------------------------------
diff --git a/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestProtoAsyncRpc.java b/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestProtoAsyncRpc.java
index 69d68b8..a7bc4d0 100644
--- a/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestProtoAsyncRpc.java
+++ b/tajo-rpc/src/test/java/org/apache/tajo/rpc/TestProtoAsyncRpc.java
@@ -27,6 +27,7 @@ import org.apache.tajo.rpc.test.TestProtos.EchoMessage;
import org.apache.tajo.rpc.test.TestProtos.SumRequest;
import org.apache.tajo.rpc.test.TestProtos.SumResponse;
import org.apache.tajo.rpc.test.impl.DummyProtocolAsyncImpl;
+import org.apache.tajo.util.NetUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -56,7 +57,8 @@ public class TestProtoAsyncRpc {
server = new ProtoAsyncRpcServer(DummyProtocol.class,
service, new InetSocketAddress(0));
server.start();
- client = new ProtoAsyncRpcClient(DummyProtocol.class, server.getListenAddress());
+ client = new ProtoAsyncRpcClient(DummyProtocol.class,
+ NetUtils.getConnectAddress(server.getListenAddress()));
stub = client.getStub();
}