You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by wl...@apache.org on 2015/11/25 11:01:17 UTC
incubator-hawq git commit: HAWQ-193. Remove libhdfs config and codes
from libyarn
Repository: incubator-hawq
Updated Branches:
refs/heads/master 7b23553a4 -> 06b7f2f05
HAWQ-193. Remove libhdfs config and codes from libyarn
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/06b7f2f0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/06b7f2f0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/06b7f2f0
Branch: refs/heads/master
Commit: 06b7f2f058d751edc182051bb7c5a356400a3662
Parents: 7b23553
Author: Wen Lin <wl...@pivotal.io>
Authored: Wed Nov 25 18:08:01 2015 +0800
Committer: Wen Lin <wl...@pivotal.io>
Committed: Wed Nov 25 18:08:01 2015 +0800
----------------------------------------------------------------------
depends/libyarn/sample/Makefile | 10 +
depends/libyarn/sample/c_client_main.c | 243 +++++++++++++++++++
depends/libyarn/sample/c_client_main_profile.c | 196 +++++++++++++++
depends/libyarn/sample/client_main.cpp | 139 +++++++++++
depends/libyarn/src/common/SessionConfig.cpp | 58 +----
depends/libyarn/src/common/SessionConfig.h | 150 +-----------
depends/libyarn/ztest/.gitignore | 3 -
depends/libyarn/ztest/Makefile | 18 --
depends/libyarn/ztest/c_client_main.c | 137 -----------
depends/libyarn/ztest/client_main.cpp | 139 -----------
depends/libyarn/ztest_c/Makefile | 10 -
depends/libyarn/ztest_c/c_client_main.c | 243 -------------------
depends/libyarn/ztest_c/c_client_main_profile.c | 196 ---------------
src/backend/utils/misc/etc/yarn-client.xml | 168 +------------
14 files changed, 598 insertions(+), 1112 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/06b7f2f0/depends/libyarn/sample/Makefile
----------------------------------------------------------------------
diff --git a/depends/libyarn/sample/Makefile b/depends/libyarn/sample/Makefile
new file mode 100644
index 0000000..bfd7a61
--- /dev/null
+++ b/depends/libyarn/sample/Makefile
@@ -0,0 +1,10 @@
+CC=gcc
+CPPFLAGS=-g -Wall -std=gnu99 -I../dist/include
+LDFLAGS=-L../dist/lib -L/Users/yjin/greenplum-db-devel/lib -lyarn
+#bin = c_client_main_profile
+bin = c_client_main
+
+all: $(bin)
+
+clean:
+ rm -rf $(bin) *.dSYM
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/06b7f2f0/depends/libyarn/sample/c_client_main.c
----------------------------------------------------------------------
diff --git a/depends/libyarn/sample/c_client_main.c b/depends/libyarn/sample/c_client_main.c
new file mode 100644
index 0000000..bf00dab
--- /dev/null
+++ b/depends/libyarn/sample/c_client_main.c
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <unistd.h>
+#include <stdio.h>
+#include <sys/time.h>
+#include "libyarn/LibYarnClientC.h"
+#include "libyarn/LibYarnConstants.h"
+
+int main() {
+ char *rmHost = "master";
+ char *rmPort = "8032";
+ char *schedHost = "master";
+ char *schedPort = "8030";
+ char *amHost = "master";
+ int32_t amPort = 8090;
+ char *am_tracking_url = "url";
+ int heartbeatInterval = 1000000;
+ int i=0;
+
+ //0. new client
+ LibYarnClient_t *client = NULL;
+ int result = newLibYarnClient(rmHost, rmPort, schedHost, schedPort,
+ amHost, amPort, am_tracking_url,&client,heartbeatInterval);
+ printf("newLibYarnClient Result Code:%d\n",result);
+
+ //1. createJob
+ char *jobName = "libyarn";
+ char *queue = "sample_queue";
+ char *jobId = NULL;
+ result = createJob(client, jobName, queue,&jobId);
+ printf("1. createJob, jobid:%s The createJob Result Code:%d\n", jobId,result);
+ if (result != FUNCTION_SUCCEEDED){
+ const char* errorMessage = getErrorMessage();
+ printf("1. createJob, errorMessage:%s\n",errorMessage);
+ }
+
+
+ char *blackListAdditions[0];
+ char *blackListRemovals[0];
+
+ //1. allocate resource
+ LibYarnResource_t *allocatedResourcesArray;
+ int allocatedResourceArraySize;
+ result = allocateResources(client, jobId, 1, 1, 1024, 5,
+ blackListAdditions, 0, blackListRemovals, 0, NULL, 0,
+ &allocatedResourcesArray, &allocatedResourceArraySize);
+ printf("The allocateResources Result Code:%d\n",result);
+ if (result != FUNCTION_SUCCEEDED){
+ const char* errorMessage = getErrorMessage();
+ printf("2. allocateResources, errorMessage:%s\n",errorMessage);
+ }
+
+ int32_t activeContainerIds[allocatedResourceArraySize];
+ int32_t releaseContainerIds[allocatedResourceArraySize];
+ int32_t statusContainerIds[allocatedResourceArraySize];
+ printf("2. allocateResources, allocatedResourceArraySize:%d\n", allocatedResourceArraySize);
+ for (i = 0 ; i < allocatedResourceArraySize; i++) {
+ puts("----------------------------");
+ printf("allocatedResourcesArray[i].containerId:%d\n", allocatedResourcesArray[i].containerId);
+ activeContainerIds[i] = allocatedResourcesArray[i].containerId;
+ releaseContainerIds[i] = allocatedResourcesArray[i].containerId;
+ statusContainerIds[i] = allocatedResourcesArray[i].containerId;
+ printf("allocatedResourcesArray[i].host:%s\n", allocatedResourcesArray[i].host);
+ printf("allocatedResourcesArray[i].port:%d\n", allocatedResourcesArray[i].port);
+ printf("allocatedResourcesArray[i].nodeHttpAddress:%s\n", allocatedResourcesArray[i].nodeHttpAddress);
+ printf("allocatedResourcesArray[i].vCores:%d\n", allocatedResourcesArray[i].vCores);
+ printf("allocatedResourcesArray[i].memory:%d\n", allocatedResourcesArray[i].memory);
+ }
+
+ //3. active
+ result = activeResources(client, jobId, activeContainerIds,allocatedResourceArraySize);
+ printf("The activeResources Result Code:%d\n",result);
+ if (result != FUNCTION_SUCCEEDED){
+ const char* errorMessage = getErrorMessage();
+ printf("2. activeResources, errorMessage:%s\n",errorMessage);
+ }
+
+ sleep(10);
+
+ int *activeFailIds;
+ int activeFailSize;
+ result = getActiveFailContainerIds(client,&activeFailIds,&activeFailSize);
+ printf("Active Fail Container Size:%d\n",activeFailSize);
+ for (i = 0;i < activeFailSize;i++){
+ printf("Active Fail Container Id:%d\n",activeFailIds[i]);
+ }
+
+ //4. getContainerReport
+ LibYarnContainerReport_t *containerReportArray;
+ int containerReportArraySize;
+ result = getContainerReports(client, jobId, &containerReportArray,
+ &containerReportArraySize);
+ printf("The getContainerReports Result Code:%d\n", result);
+ if (result != FUNCTION_SUCCEEDED) {
+ const char* errorMessage = getErrorMessage();
+ printf(" The getContainerReports, errorMessage:%s\n", errorMessage);
+ }
+
+ printf("containerReportArraySize=%d\n", containerReportArraySize);
+ for (i = 0; i < containerReportArraySize; i++) {
+ printf("-------------container: %d--------------------------\n", i);
+ printf("containerId:%d\n", containerReportArray[i].containerId);
+ printf("vCores:%d\n", containerReportArray[i].vCores);
+ printf("memory:%d\n", containerReportArray[i].memory);
+ printf("host:%s\n", containerReportArray[i].host);
+ printf("port:%d\n", containerReportArray[i].port);
+ printf("exitStatus:%d\n", containerReportArray[i].exitStatus);
+ printf("state:%d\n", containerReportArray[i].state);
+ }
+ freeContainerReportArray(containerReportArray, containerReportArraySize);
+
+ //4. getContainerReport
+ LibYarnContainerStatus_t *containerStatusArray;
+ int containerStatusArraySize;
+ result = getContainerStatuses(client, jobId, statusContainerIds,
+ allocatedResourceArraySize, &containerStatusArray,
+ &containerStatusArraySize);
+ printf("The getContainerStatus Result Code:%d\n", result);
+ if (result != FUNCTION_SUCCEEDED) {
+ const char* errorMessage = getErrorMessage();
+ printf(" The getContainerStatus, errorMessage:%s\n", errorMessage);
+ }
+
+ printf("containerStatusArraySize=%d\n", containerStatusArraySize);
+ for (i = 0; i < containerStatusArraySize; i++) {
+ printf("-------------container: %d--------------------------\n", i);
+ printf("containerId:%d\n", containerStatusArray[i].containerId);
+ printf("exitStatus:%d\n", containerStatusArray[i].exitStatus);
+ printf("state:%d\n", containerStatusArray[i].state);
+ printf("diagnostics:%s\n", containerStatusArray[i].diagnostics);
+ }
+ freeContainerStatusArray(containerStatusArray,containerStatusArraySize);
+
+ //6. getQueueInfo
+ LibYarnQueueInfo_t *queueInfo = NULL;
+ result = getQueueInfo(client, queue, true, true, true, &queueInfo);
+ printf("The getQueueInfo Result Code:%d\n",result);
+ if (result != FUNCTION_SUCCEEDED){
+ const char* errorMessage = getErrorMessage();
+ printf(" The getQueueInfo, errorMessage:%s\n",errorMessage);
+ }
+
+ printf("QueueInfo: queueInfo->queueName:%s, queueInfo->capacity:%f, queueInfo->maximumCapacity:%f, queueInfo->currentCapacity:%f, queueInfo->state:%d\n",
+ queueInfo->queueName, queueInfo->capacity, queueInfo->maximumCapacity,
+ queueInfo->currentCapacity, queueInfo->state);
+ puts("---------chilldQueue:");
+ for (i = 0; i < queueInfo->childQueueNameArraySize; i++) {
+ printf("QueueInfo: queueInfo->childQueueNameArray[%d]:%s\n", i, queueInfo->childQueueNameArray[i]);
+ }
+ freeMemQueueInfo(queueInfo);
+
+ //7. getCluster
+ LibYarnNodeReport_t *nodeReportArray;
+ int nodeReportArraySize;
+ result = getClusterNodes(client, NODE_STATE_RUNNING, &nodeReportArray, &nodeReportArraySize);
+ printf("The getClusterNodes Result Code:%d\n",result);
+ if (result != FUNCTION_SUCCEEDED){
+ const char* errorMessage = getErrorMessage();
+ printf(" The getClusterNodes, errorMessage:%s\n",errorMessage);
+ }
+
+ printf("nodeReportArraySize=%d\n", nodeReportArraySize);
+ for (i = 0; i < nodeReportArraySize; i++) {
+ printf("-------------node %d--------------------------\n", i);
+ printf("host:%s\n", nodeReportArray[i].host);
+ printf("port:%d\n", nodeReportArray[i].port);
+ printf("httpAddress:%s\n", nodeReportArray[i].httpAddress);
+ printf("rackName:%s\n", nodeReportArray[i].rackName);
+ printf("memoryUsed:%d\n", nodeReportArray[i].memoryUsed);
+ printf("vcoresUsed:%d\n", nodeReportArray[i].vcoresUsed);
+ printf("memoryCapability:%d\n", nodeReportArray[i].memoryCapability);
+ printf("vcoresCapability:%d\n", nodeReportArray[i].vcoresCapability);
+ printf("numContainers:%d\n", nodeReportArray[i].numContainers);
+ printf("nodeState:%d\n", nodeReportArray[i].nodeState);
+ printf("healthReport:%s\n", nodeReportArray[i].healthReport);
+ printf("lastHealthReportTime:%lld\n", nodeReportArray[i].lastHealthReportTime);
+ }
+ freeMemNodeReportArray(nodeReportArray, nodeReportArraySize);
+
+ //8. getApplicationReport
+ LibYarnApplicationReport_t *applicationReport = NULL;
+ result = getApplicationReport(client, jobId, &applicationReport);
+ printf("The getApplicationReport Result Code:%d\n", result);
+ if (result != FUNCTION_SUCCEEDED) {
+ const char* errorMessage = getErrorMessage();
+ printf(" The getApplicationReport, errorMessage:%s\n",
+ errorMessage);
+ }
+ printf("-------------ApplicationReport-------------------------\n");
+ printf("appId:%d\n", applicationReport->appId);
+ printf("user:%s\n", applicationReport->user);
+ printf("queue:%s\n", applicationReport->queue);
+ printf("name:%s\n", applicationReport->name);
+ printf("host:%s\n", applicationReport->host);
+ printf("port:%d\n", applicationReport->port);
+ printf("status:%d\n", applicationReport->status);
+
+ freeApplicationReport(applicationReport);
+
+ //5. release
+ result = releaseResources(client, jobId, releaseContainerIds,allocatedResourceArraySize);
+ printf("The releaseResources Result Code:%d\n",result);
+
+ result = getActiveFailContainerIds(client,&activeFailIds,&activeFailSize);
+ printf("Active Fail Container Size:%d\n",activeFailSize);
+ for (i = 0;i < activeFailSize;i++){
+ printf("Active Fail Container Id:%d\n",activeFailIds[i]);
+ }
+ free(activeFailIds);
+ freeMemAllocatedResourcesArray(allocatedResourcesArray, allocatedResourceArraySize);
+ printf("freeMemAllocated is OK\n");
+
+ //9. finish
+ printf("jobId:%s\n", jobId);
+ result = finishJob(client, jobId, APPLICATION_SUCCEEDED);
+ printf("The finishJob Result Code:%d\n", result);
+ if (result != FUNCTION_SUCCEEDED) {
+ const char* errorMessage = getErrorMessage();
+ printf(" The finishJob, errorMessage:%s\n", errorMessage);
+ }
+ free(jobId);
+ //10. free client
+ deleteLibYarnClient(client);
+
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/06b7f2f0/depends/libyarn/sample/c_client_main_profile.c
----------------------------------------------------------------------
diff --git a/depends/libyarn/sample/c_client_main_profile.c b/depends/libyarn/sample/c_client_main_profile.c
new file mode 100644
index 0000000..b94c919
--- /dev/null
+++ b/depends/libyarn/sample/c_client_main_profile.c
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <unistd.h>
+#include <stdio.h>
+#include <sys/time.h>
+#include "libyarn/LibYarnClientC.h"
+#include "libyarn/LibYarnConstants.h"
+
+int main() {
+ struct timeval tpstart,tpend;
+ float allocateTime = 0;
+ float activeTime = 0;
+ float releaseTime = 0;
+ float timeuse;
+ char *rmHost = "localhost";
+ char *rmPort = "8032";
+ char *schedHost = "localhost";
+ char *schedPort = "8030";
+ char *amHost = "localhost";
+ int32_t amPort = 10;
+ char *am_tracking_url = "url";
+ int heartbeatInterval = 1000;
+
+ //0. new client
+ LibYarnClient_t *client = NULL;
+ int result = newLibYarnClient(rmHost, rmPort, schedHost, schedPort,
+ amHost, amPort, am_tracking_url,&client,heartbeatInterval);
+
+ //1. createJob
+ char *jobName = "libyarn";
+ char *queue = "default";
+ char *jobId = NULL;
+ result = createJob(client, jobName, queue,&jobId);
+ if (result != FUNCTION_SUCCEEDED){
+ const char* errorMessage = getErrorMessage();
+ printf("1. createJob, errorMessage:%s\n",errorMessage);
+ }
+
+ int i,j;
+ for (j = 0;j < 10;j++){
+ LibYarnResourceRequest_t resRequest;
+ resRequest.priority = 1;
+ resRequest.host = "*";
+ resRequest.vCores = 1;
+ resRequest.memory = 1024;
+ resRequest.num_containers = 2;
+ resRequest.relax_locality = 1;
+
+ char *blackListAdditions[0];
+ char *blackListRemovals[0];
+
+ //1. allocate resource
+ LibYarnResource_t *allocatedResourcesArray;
+ int allocatedResourceArraySize;
+ gettimeofday(&tpstart,NULL);
+ result = allocateResources(client, jobId, &resRequest, blackListAdditions,
+ 0, blackListRemovals, 0, &allocatedResourcesArray, &allocatedResourceArraySize,5);
+ gettimeofday(&tpend,NULL);
+ timeuse=1000000*(tpend.tv_sec-tpstart.tv_sec)+tpend.tv_usec-tpstart.tv_usec;
+ allocateTime += timeuse;
+
+ if (result != FUNCTION_SUCCEEDED){
+ const char* errorMessage = getErrorMessage();
+ printf("2. allocateResources, errorMessage:%s\n",errorMessage);
+ }
+
+ int32_t activeContainerIds[allocatedResourceArraySize];
+ int32_t releaseContainerIds[allocatedResourceArraySize];
+ int32_t statusContainerIds[allocatedResourceArraySize];
+ for (i = 0 ; i < allocatedResourceArraySize; i++) {
+ activeContainerIds[i] = allocatedResourcesArray[i].containerId;
+ releaseContainerIds[i] = allocatedResourcesArray[i].containerId;
+ statusContainerIds[i] = allocatedResourcesArray[i].containerId;
+ }
+
+ //3. active
+ gettimeofday(&tpstart,NULL);
+ result = activeResources(client, jobId, activeContainerIds,allocatedResourceArraySize);
+ gettimeofday(&tpend,NULL);
+ timeuse=1000000*(tpend.tv_sec-tpstart.tv_sec)+tpend.tv_usec-tpstart.tv_usec;
+ activeTime += timeuse;
+ if (result != FUNCTION_SUCCEEDED){
+ const char* errorMessage = getErrorMessage();
+ printf("2. activeResources, errorMessage:%s\n",errorMessage);
+ }
+
+ int *activeFailIds;
+ int activeFailSize;
+ result = getActiveFailContainerIds(client,&activeFailIds,&activeFailSize);
+ for (i = 0;i < activeFailSize;i++){
+ printf("Active Fail Container Id:%d\n",activeFailIds[i]);
+ }
+ //sleep(160);
+
+ //4. getContainerReport
+ LibYarnContainerReport_t *containerReportArray;
+ int containerReportArraySize;
+ result = getContainerReports(client, jobId, &containerReportArray,
+ &containerReportArraySize);
+ if (result != FUNCTION_SUCCEEDED) {
+ const char* errorMessage = getErrorMessage();
+ printf(" The getContainerReports, errorMessage:%s\n", errorMessage);
+ }
+
+ freeContainerReportArray(containerReportArray, containerReportArraySize);
+
+ //4. getContainerReport
+ LibYarnContainerStatus_t *containerStatusArray;
+ int containerStatusArraySize;
+ result = getContainerStatuses(client, jobId, statusContainerIds,
+ allocatedResourceArraySize, &containerStatusArray,
+ &containerStatusArraySize);
+ if (result != FUNCTION_SUCCEEDED) {
+ const char* errorMessage = getErrorMessage();
+ printf(" The getContainerStatus, errorMessage:%s\n", errorMessage);
+ }
+ freeContainerStatusArray(containerStatusArray,containerStatusArraySize);
+
+ //5. release
+ gettimeofday(&tpstart,NULL);
+ result = releaseResources(client, jobId, releaseContainerIds,allocatedResourceArraySize);
+ gettimeofday(&tpend,NULL);
+ timeuse=1000000*(tpend.tv_sec-tpstart.tv_sec)+tpend.tv_usec-tpstart.tv_usec;
+ releaseTime += timeuse;
+
+ result = getActiveFailContainerIds(client,&activeFailIds,&activeFailSize);
+ for (i = 0;i < activeFailSize;i++){
+ printf("Active Fail Container Id:%d\n",activeFailIds[i]);
+ }
+ free(activeFailIds);
+ freeMemAllocatedResourcesArray(allocatedResourcesArray, allocatedResourceArraySize);
+
+ //6. getQueueInfo
+ LibYarnQueueInfo_t *queueInfo = NULL;
+ result = getQueueInfo(client, queue, true, true, true, &queueInfo);
+ if (result != FUNCTION_SUCCEEDED){
+ const char* errorMessage = getErrorMessage();
+ printf(" The getQueueInfo, errorMessage:%s\n",errorMessage);
+ }
+
+ freeMemQueueInfo(queueInfo);
+
+ //7. getCluster
+ LibYarnNodeReport_t *nodeReportArray;
+ int nodeReportArraySize;
+ result = getClusterNodes(client, NODE_STATE_RUNNING, &nodeReportArray, &nodeReportArraySize);
+ if (result != FUNCTION_SUCCEEDED){
+ const char* errorMessage = getErrorMessage();
+ printf(" The getClusterNodes, errorMessage:%s\n",errorMessage);
+ }
+
+ freeMemNodeReportArray(nodeReportArray, nodeReportArraySize);
+
+ //8. getApplicationReport
+ LibYarnApplicationReport_t *applicationReport = NULL;
+ result = getApplicationReport(client, jobId, &applicationReport);
+ if (result != FUNCTION_SUCCEEDED) {
+ const char* errorMessage = getErrorMessage();
+ printf(" The getApplicationReport, errorMessage:%s\n",
+ errorMessage);
+ }
+
+ freeApplicationReport(applicationReport);
+ }
+ printf("allocateTime:%f s\n",allocateTime/1000000);
+ printf("activeTime:%f s\n",activeTime/1000000);
+ printf("releaseTime:%f s\n",releaseTime/1000000);
+ //9. finish
+ result = finishJob(client, jobId, APPLICATION_SUCCEEDED);
+ if (result != FUNCTION_SUCCEEDED) {
+ const char* errorMessage = getErrorMessage();
+ printf(" The finishJob, errorMessage:%s\n", errorMessage);
+ }
+ free(jobId);
+ //10. free client
+ deleteLibYarnClient(client);
+
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/06b7f2f0/depends/libyarn/sample/client_main.cpp
----------------------------------------------------------------------
diff --git a/depends/libyarn/sample/client_main.cpp b/depends/libyarn/sample/client_main.cpp
new file mode 100644
index 0000000..24601b7
--- /dev/null
+++ b/depends/libyarn/sample/client_main.cpp
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <iostream>
+#include <list>
+
+#include <unistd.h>
+
+#include "libyarn/LibYarnClient.h"
+#include "libyarn/records/FinalApplicationStatus.h"
+
+using namespace std;
+using namespace libyarn;
+
+int main(void)
+{
+
+ string rmHost("localhost");
+ string rmPort("8032");
+ string schedHost("localhost");
+ string schedPort("8030");
+ string amHost("localhost");
+ int32_t amPort = 0;
+
+
+ //cluster node1
+/*
+ string rmHost("10.37.7.101");
+ string rmPort("8032");
+ string schedHost("10.37.7.101");
+ string schedPort("8030");
+ string amHost("10.34.0.134");
+ int32_t amPort = 0;
+*/
+
+ string am_tracking_url("url");
+ LibYarnClient client(rmHost, rmPort, schedHost, schedPort, amHost, amPort, am_tracking_url);
+
+ //1. createJob
+ string jobName("libyarn");
+ string queue("default");
+ string jobId = client.createJob(jobName, queue);
+ cout<<"jobId:"<<jobId<<endl;
+
+ //2. allocate
+ ResourceRequest resRequest;
+ string host("*");
+ resRequest.setResourceName(host);
+
+ Resource capability;
+ capability.setVirtualCores(1);
+ capability.setMemory(1024);
+ resRequest.setCapability(capability);
+
+ resRequest.setNumContainers(3);
+
+ resRequest.setRelaxLocality(true);
+
+ Priority priority;
+ priority.setPriority(1);
+ resRequest.setPriority(priority);
+
+
+ string resGroupId("group_1");
+ list<string> blackListAdditions;
+ list<string> blackListRemovals;
+
+ list<Container> allocatedResources = client.allocateResources(jobId,
+ resGroupId, resRequest, blackListAdditions, blackListRemovals);
+
+ for (list<Container>::iterator it = allocatedResources.begin();
+ it != allocatedResources.end(); it++) {
+ cout << "allocate: host:" << (*it).getNodeId().getHost() << ", port:"
+ << (*it).getNodeId().getPort() << ", cid:" << (*it).getId().getId()
+ << ", vcores:" << (*it).getResource().getVirtualCores() << ", mem:"
+ << (*it).getResource().getMemory() << endl;
+ }
+
+ //3. active
+ client.activeResources(jobId, resGroupId);
+
+ sleep(1);
+
+ //4. release
+ client.releaseResources(jobId, resGroupId);
+
+ sleep(2);
+
+ //5. getQueueInfo
+ QueueInfo queueInfo = client.getQueueInfo(queue, true, true, true);
+ cout << "queueName:" << queueInfo.getQueueName() << ", capacity:"
+ << queueInfo.getCapacity() << ", maximumCapacity:"
+ << queueInfo.getMaximumCapacity() << ", currentCapacity:"
+ << queueInfo.getCurrentCapacity() << ", state:"
+ << queueInfo.getQueueState() << endl;
+
+ //6. getCluster
+ list<NodeState> nodeStates;
+ nodeStates.push_back(NodeState::NS_RUNNING);
+ list<NodeReport> nodeReports = client.getClusterNodes(nodeStates);
+ for (list<NodeReport>::iterator it = nodeReports.begin(); it != nodeReports.end(); it++) {
+ cout << "host:" << it->getNodeId().getHost() << ", port:"
+ << it->getNodeId().getPort() << ", httpAddress:" << it->getHttpAddress()
+ << ", rackName:" << it->getRackName() << ", used:[vcore:"
+ << it->getUsedResource().getVirtualCores() << ", mem:"
+ << it->getUsedResource().getMemory() << "], capability:[vcore:"
+ << it->getResourceCapability().getVirtualCores() << ", mem:"
+ << it->getResourceCapability().getMemory() << "], numContainers:"
+ << it->getNumContainers() << ", node_state:" << it->getNodeState()
+ << ", health_report:" << it->getHealthReport()
+ << ", last_health_report_time:" << it->getLastHealthReportTime()
+ << endl;
+ }
+
+ //7. finish
+ cout << "to finish: jobId:" << jobId << endl;
+ client.finishJob(jobId, FinalApplicationStatus::APP_SUCCEEDED);
+
+ sleep(1);
+
+
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/06b7f2f0/depends/libyarn/src/common/SessionConfig.cpp
----------------------------------------------------------------------
diff --git a/depends/libyarn/src/common/SessionConfig.cpp b/depends/libyarn/src/common/SessionConfig.cpp
index 43b0654..33be380 100644
--- a/depends/libyarn/src/common/SessionConfig.cpp
+++ b/depends/libyarn/src/common/SessionConfig.cpp
@@ -50,10 +50,6 @@ SessionConfig::SessionConfig(const Config & conf) {
ConfigDefault<bool> boolValues [] = {
{
&rpcTcpNoDelay, "rpc.client.connect.tcpnodelay", true
- }, {
- &readFromLocal, "dfs.client.read.shortcircuit", true
- }, {
- &addDatanode, "output.replace-datanode-on-failure", true
}
};
ConfigDefault<int32_t> i32Values[] = {
@@ -74,53 +70,14 @@ SessionConfig::SessionConfig(const Config & conf) {
}, {
&rpcTimeout, "rpc.client.timeout", 3600 * 1000
}, {
- &defaultReplica, "dfs.default.replica", 3, bind(CheckRangeGE<int32_t>, _1, _2, 1)
- }, {
- &inputConnTimeout, "input.connect.timeout", 600 * 1000
- }, {
- &inputReadTimeout, "input.read.timeout", 3600 * 1000
- }, {
- &inputWriteTimeout, "input.write.timeout", 3600 * 1000
- }, {
- &localReadBufferSize, "input.localread.default.buffersize", 1 * 1024 * 1024, bind(CheckRangeGE<int32_t>, _1, _2, 1)
- }, {
- &prefetchSize, "dfs.prefetchsize", 10, bind(CheckRangeGE<int32_t>, _1, _2, 1)
- }, {
- &maxGetBlockInfoRetry, "input.read.getblockinfo.retry", 3, bind(CheckRangeGE<int32_t>, _1, _2, 1)
- }, {
- &maxLocalBlockInfoCacheSize, "input.localread.blockinfo.cachesize", 1000, bind(CheckRangeGE<int32_t>, _1, _2, 1)
- }, {
- &chunkSize, "output.default.chunksize", 512, bind(CheckMultipleOf<int32_t>, _1, _2, 512)
- }, {
- &packetSize, "output.default.packetsize", 64 * 1024
- }, {
- &blockWriteRetry, "output.default.write.retry", 10, bind(CheckRangeGE<int32_t>, _1, _2, 1)
- }, {
- &outputConnTimeout, "output.connect.timeout", 600 * 1000
- }, {
- &outputReadTimeout, "output.read.timeout", 3600 * 1000
- }, {
- &outputWriteTimeout, "output.write.timeout", 3600 * 1000
- }, {
- &closeFileTimeout, "output.close.timeout", 3600 * 1000
- }, {
- &packetPoolSize, "output.packetpool.size", 1024
- }, {
- &heartBeatInterval, "output.heeartbeat.interval", 10 * 1000
- }, {
- &rpcMaxHARetry, "dfs.client.failover.max.attempts", 15, bind(CheckRangeGE<int32_t>, _1, _2, 0)
- }
- };
- ConfigDefault<int64_t> i64Values [] = {
- {
- &defaultBlockSize, "dfs.default.blocksize", 64 * 1024 * 1024, bind(CheckMultipleOf<int64_t>, _1, _2, 512)
+ &rpcMaxHARetry, "yarn.client.failover.max.attempts", 15, bind(CheckRangeGE<int32_t>, _1, _2, 0)
}
};
+
ConfigDefault<std::string> strValues [] = {
- {&defaultUri, "dfs.default.uri", "hdfs://localhost:9000" },
{&rpcAuthMethod, "hadoop.security.authentication", "simple" },
{&kerberosCachePath, "hadoop.security.kerberos.ticket.cache.path", "" },
- {&logSeverity, "dfs.client.log.severity", "INFO" }
+ {&logSeverity, "yarn.client.log.severity", "INFO" }
};
for (size_t i = 0; i < ARRAYSIZE(boolValues); ++i) {
@@ -141,15 +98,6 @@ SessionConfig::SessionConfig(const Config & conf) {
}
}
- for (size_t i = 0; i < ARRAYSIZE(i64Values); ++i) {
- *i64Values[i].variable = conf.getInt64(i64Values[i].key,
- i64Values[i].value);
-
- if (i64Values[i].check) {
- i64Values[i].check(i64Values[i].key, *i64Values[i].variable);
- }
- }
-
for (size_t i = 0; i < ARRAYSIZE(strValues); ++i) {
*strValues[i].variable = conf.getString(strValues[i].key,
strValues[i].value);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/06b7f2f0/depends/libyarn/src/common/SessionConfig.h
----------------------------------------------------------------------
diff --git a/depends/libyarn/src/common/SessionConfig.h b/depends/libyarn/src/common/SessionConfig.h
index 1459134..1b56e91 100644
--- a/depends/libyarn/src/common/SessionConfig.h
+++ b/depends/libyarn/src/common/SessionConfig.h
@@ -17,8 +17,8 @@
* under the License.
*/
-#ifndef _HDFS_LIBHDFS3_COMMON_SESSIONCONFIG_H_
-#define _HDFS_LIBHDFS3_COMMON_SESSIONCONFIG_H_
+#ifndef _YARN_LIBYARN_COMMON_SESSIONCONFIG_H_
+#define _YARN_LIBYARN_COMMON_SESSIONCONFIG_H_
#include "Exception.h"
#include "ExceptionInternal.h"
@@ -77,105 +77,14 @@ public:
return rpcWriteTimeout;
}
- /*
- * FileSystem configure
- */
- const std::string & getDefaultUri() const {
- return defaultUri;
- }
-
- int32_t getDefaultReplica() const {
- return defaultReplica;
- }
-
- int64_t getDefaultBlockSize() const {
- return defaultBlockSize;
- }
-
- /*
- * InputStream configure
- */
- int32_t getLocalReadBufferSize() const {
- return localReadBufferSize;
- }
-
- int32_t getInputReadTimeout() const {
- return inputReadTimeout;
- }
-
- int32_t getInputWriteTimeout() const {
- return inputWriteTimeout;
- }
-
- int32_t getInputConnTimeout() const {
- return inputConnTimeout;
- }
-
- int32_t getPrefetchSize() const {
- return prefetchSize;
- }
-
- bool isReadFromLocal() const {
- return readFromLocal;
- }
-
- int32_t getMaxGetBlockInfoRetry() const {
- return maxGetBlockInfoRetry;
- }
-
- int32_t getMaxLocalBlockInfoCacheSize() const {
- return maxLocalBlockInfoCacheSize;
- }
-
- /*
- * OutputStream configure
- */
- int32_t getDefaultChunkSize() const {
- return chunkSize;
- }
-
- int32_t getDefaultPacketSize() const {
- if (packetSize % chunkSize != 0) {
- THROW(YarnConfigInvalid,
- "output.default.packetsize should be larger than 0 "
- "and be the multiple of output.default.chunksize.");
- }
-
- return packetSize;
- }
-
- int32_t getBlockWriteRetry() const {
- return blockWriteRetry;
- }
-
- int32_t getOutputConnTimeout() const {
- return outputConnTimeout;
- }
-
- int32_t getOutputReadTimeout() const {
- return outputReadTimeout;
- }
-
- int32_t getOutputWriteTimeout() const {
- return outputWriteTimeout;
- }
-
- bool canAddDatanode() const {
- return addDatanode;
- }
-
- int32_t getHeartBeatInterval() const {
- return heartBeatInterval;
+ void setRpcMaxHaRetry(int32_t rpcMaxHaRetry) {
+ rpcMaxHARetry = rpcMaxHaRetry;
}
int32_t getRpcMaxHaRetry() const {
return rpcMaxHARetry;
}
- void setRpcMaxHaRetry(int32_t rpcMaxHaRetry) {
- rpcMaxHARetry = rpcMaxHaRetry;
- }
-
const std::string & getRpcAuthMethod() const {
return rpcAuthMethod;
}
@@ -215,22 +124,6 @@ public:
this->logSeverity = logSeverityLevel;
}
- int32_t getPacketPoolSize() const {
- return packetPoolSize;
- }
-
- void setPacketPoolSize(int32_t packetPoolSize) {
- this->packetPoolSize = packetPoolSize;
- }
-
- int32_t getCloseFileTimeout() const {
- return closeFileTimeout;
- }
-
- void setCloseFileTimeout(int32_t closeFileTimeout) {
- this->closeFileTimeout = closeFileTimeout;
- }
-
int32_t getRpcTimeout() const {
return rpcTimeout;
}
@@ -255,44 +148,11 @@ public:
bool rpcTcpNoDelay;
std::string rpcAuthMethod;
- /*
- * FileSystem configure
- */
- std::string defaultUri;
std::string kerberosCachePath;
std::string logSeverity;
- int32_t defaultReplica;
- int64_t defaultBlockSize;
-
- /*
- * InputStream configure
- */
- bool readFromLocal;
- int32_t inputConnTimeout;
- int32_t inputReadTimeout;
- int32_t inputWriteTimeout;
- int32_t localReadBufferSize;
- int32_t maxGetBlockInfoRetry;
- int32_t maxLocalBlockInfoCacheSize;
- int32_t prefetchSize;
-
- /*
- * OutputStream configure
- */
- bool addDatanode;
- int32_t chunkSize;
- int32_t packetSize;
- int32_t blockWriteRetry; //retry on block not replicated yet.
- int32_t outputConnTimeout;
- int32_t outputReadTimeout;
- int32_t outputWriteTimeout;
- int32_t packetPoolSize;
- int32_t heartBeatInterval;
- int32_t closeFileTimeout;
-
};
}
}
-#endif /* _HDFS_LIBHDFS3_COMMON_SESSIONCONFIG_H_ */
+#endif /* _YARN_LIBYARN_COMMON_SESSIONCONFIG_H_ */
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/06b7f2f0/depends/libyarn/ztest/.gitignore
----------------------------------------------------------------------
diff --git a/depends/libyarn/ztest/.gitignore b/depends/libyarn/ztest/.gitignore
deleted file mode 100644
index 4a11725..0000000
--- a/depends/libyarn/ztest/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-*_main
-*.dSYM
-
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/06b7f2f0/depends/libyarn/ztest/Makefile
----------------------------------------------------------------------
diff --git a/depends/libyarn/ztest/Makefile b/depends/libyarn/ztest/Makefile
deleted file mode 100644
index ba4b871..0000000
--- a/depends/libyarn/ztest/Makefile
+++ /dev/null
@@ -1,18 +0,0 @@
-CC=g++
-CPPFLAGS=-g -Wall -std=c++0x -I/Users/jcao/program/libyarn/include -Wno-write-strings
-LDFLAGS=-L/Users/jcao/program/libyarn/lib -lyarn
-
-
-bin = client_main c_client_main
-
-all: $(bin)
-
-test_main: test_main.cpp
- $(CC) $(CPPFLAGS) $(LDFLAGS) $^ -o $@
-
-client_main: client_main.cpp
- $(CC) $(CPPFLAGS) $(LDFLAGS) $^ -o $@
-
-
-clean:
- rm -rf $(bin) *.dSYM
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/06b7f2f0/depends/libyarn/ztest/c_client_main.c
----------------------------------------------------------------------
diff --git a/depends/libyarn/ztest/c_client_main.c b/depends/libyarn/ztest/c_client_main.c
deleted file mode 100644
index 6ba35a6..0000000
--- a/depends/libyarn/ztest/c_client_main.c
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include <unistd.h>
-#include "libyarn/LibYarnClientC.h"
-#include "libyarn/LibYarnConstants.h"
-
-int main() {
- char *rmHost = "localhost";
- char *rmPort = "8032";
- char *schedHost = "localhost";
- char *schedPort = "8030";
- char *amHost = "localhost";
- int32_t amPort = 0;
- char *am_tracking_url = "url";
-
- int i;
-
- //0. new client
- LibYarnClient_t *client = newLibYarnClient(rmHost, rmPort, schedHost, schedPort,
- amHost, amPort, am_tracking_url);
-
- //1. createJob
- char *jobName = "libyarn";
- char *queue = "default";
- char* jobId = createJob(client, jobName, queue);
- printf("1. createJob, jobid:%s\n", jobId);
-
- //2. allocate
- char *resGroupId = "group_1";
- LibYarnResourceRequest_t resRequest;
- resRequest.priority = 1;
- resRequest.host = "*";
- resRequest.vCores = 1;
- resRequest.memory = 1024;
- resRequest.num_containers = 3;
- resRequest.relax_locality = 1;
-
- char *blackListAdditions[0];
- char *blackListRemovals[0];
-
- LibYarnResource_t *allocatedResourcesArray;
- int allocatedResourceArraySize;
- allocateResources(client, jobId, resGroupId, &resRequest, blackListAdditions,
- 0, blackListRemovals, 0, &allocatedResourcesArray, &allocatedResourceArraySize);
-
-
- printf("2. allocateResources, allocatedResourceArraySize:%d\n", allocatedResourceArraySize);
- for (i = 0 ; i < allocatedResourceArraySize; i++) {
- puts("----------------------------");
- printf("allocatedResourcesArray[i].containerId:%d\n", allocatedResourcesArray[i].containerId);
- printf("allocatedResourcesArray[i].host:%s\n", allocatedResourcesArray[i].host);
- printf("allocatedResourcesArray[i].port:%d\n", allocatedResourcesArray[i].port);
- printf("allocatedResourcesArray[i].nodeHttpAddress:%s\n", allocatedResourcesArray[i].nodeHttpAddress);
-
- printf("allocatedResourcesArray[i].vCores:%d\n", allocatedResourcesArray[i].vCores);
- printf("allocatedResourcesArray[i].memory:%d\n", allocatedResourcesArray[i].memory);
- }
-
- freeMemAllocatedResourcesArray(allocatedResourcesArray, allocatedResourceArraySize);
-
- //3. active
- activeResources(client, jobId, resGroupId);
-
- sleep(1);
-
- //4. release
- releaseResources(client, jobId, resGroupId);
-
-
- //5. getQueueInfo
- LibYarnQueueInfo_t *queueInfo = NULL;
- getQueueInfo(client, queue, true, true, true, &queueInfo);
- printf(
- "QueueInfo: queueInfo->queueName:%s, queueInfo->capacity:%f, queueInfo->maximumCapacity:%f, queueInfo->currentCapacity:%f, queueInfo->state:%d\n",
- queueInfo->queueName, queueInfo->capacity, queueInfo->maximumCapacity,
- queueInfo->currentCapacity, queueInfo->state);
- puts("---------chilldQueue:");
- for (i = 0; i < queueInfo->childQueueNameArraySize; i++) {
- printf("QueueInfo: queueInfo->childQueueNameArray[%d]:%s\n", i, queueInfo->childQueueNameArray[i]);
- }
- freeMemQueueInfo(queueInfo);
-
-
- //6. getCluster
- LibYarnNodeReport_t *nodeReportArray;
- int nodeReportArraySize;
- getClusterNodes(client, NODE_STATE_RUNNING, &nodeReportArray, &nodeReportArraySize);
-
- printf("nodeReportArraySize=%d\n", nodeReportArraySize);
- for (i = 0; i < nodeReportArraySize; i++) {
- printf("-------------node %d--------------------------\n", i);
- printf("host:%s\n", nodeReportArray[i].host);
- printf("port:%d\n", nodeReportArray[i].port);
- printf("httpAddress:%s\n", nodeReportArray[i].httpAddress);
- printf("rackName:%s\n", nodeReportArray[i].rackName);
- printf("memoryUsed:%d\n", nodeReportArray[i].memoryUsed);
- printf("vcoresUsed:%d\n", nodeReportArray[i].vcoresUsed);
- printf("memoryCapability:%d\n", nodeReportArray[i].memoryCapability);
- printf("vcoresCapability:%d\n", nodeReportArray[i].vcoresCapability);
- printf("numContainers:%d\n", nodeReportArray[i].numContainers);
- printf("nodeState:%d\n", nodeReportArray[i].nodeState);
- printf("healthReport:%s\n", nodeReportArray[i].healthReport);
- printf("lastHealthReportTime:%lld\n", nodeReportArray[i].lastHealthReportTime);
- }
-
- freeMemNodeReportArray(nodeReportArray, nodeReportArraySize);
-
- //7. finish
- printf("jobId:%s\n", jobId);
- finishJob(client, jobId, APPLICATION_SUCCEEDED);
-
-
- //8. free client
- deleteLibYarnClient(client);
-
- //9. free others
- free(jobId);
-
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/06b7f2f0/depends/libyarn/ztest/client_main.cpp
----------------------------------------------------------------------
diff --git a/depends/libyarn/ztest/client_main.cpp b/depends/libyarn/ztest/client_main.cpp
deleted file mode 100644
index 24601b7..0000000
--- a/depends/libyarn/ztest/client_main.cpp
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include <iostream>
-#include <list>
-
-#include <unistd.h>
-
-#include "libyarn/LibYarnClient.h"
-#include "libyarn/records/FinalApplicationStatus.h"
-
-using namespace std;
-using namespace libyarn;
-
-int main(void)
-{
-
- string rmHost("localhost");
- string rmPort("8032");
- string schedHost("localhost");
- string schedPort("8030");
- string amHost("localhost");
- int32_t amPort = 0;
-
-
- //cluster node1
-/*
- string rmHost("10.37.7.101");
- string rmPort("8032");
- string schedHost("10.37.7.101");
- string schedPort("8030");
- string amHost("10.34.0.134");
- int32_t amPort = 0;
-*/
-
- string am_tracking_url("url");
- LibYarnClient client(rmHost, rmPort, schedHost, schedPort, amHost, amPort, am_tracking_url);
-
- //1. createJob
- string jobName("libyarn");
- string queue("default");
- string jobId = client.createJob(jobName, queue);
- cout<<"jobId:"<<jobId<<endl;
-
- //2. allocate
- ResourceRequest resRequest;
- string host("*");
- resRequest.setResourceName(host);
-
- Resource capability;
- capability.setVirtualCores(1);
- capability.setMemory(1024);
- resRequest.setCapability(capability);
-
- resRequest.setNumContainers(3);
-
- resRequest.setRelaxLocality(true);
-
- Priority priority;
- priority.setPriority(1);
- resRequest.setPriority(priority);
-
-
- string resGroupId("group_1");
- list<string> blackListAdditions;
- list<string> blackListRemovals;
-
- list<Container> allocatedResources = client.allocateResources(jobId,
- resGroupId, resRequest, blackListAdditions, blackListRemovals);
-
- for (list<Container>::iterator it = allocatedResources.begin();
- it != allocatedResources.end(); it++) {
- cout << "allocate: host:" << (*it).getNodeId().getHost() << ", port:"
- << (*it).getNodeId().getPort() << ", cid:" << (*it).getId().getId()
- << ", vcores:" << (*it).getResource().getVirtualCores() << ", mem:"
- << (*it).getResource().getMemory() << endl;
- }
-
- //3. active
- client.activeResources(jobId, resGroupId);
-
- sleep(1);
-
- //4. release
- client.releaseResources(jobId, resGroupId);
-
- sleep(2);
-
- //5. getQueueInfo
- QueueInfo queueInfo = client.getQueueInfo(queue, true, true, true);
- cout << "queueName:" << queueInfo.getQueueName() << ", capacity:"
- << queueInfo.getCapacity() << ", maximumCapacity:"
- << queueInfo.getMaximumCapacity() << ", currentCapacity:"
- << queueInfo.getCurrentCapacity() << ", state:"
- << queueInfo.getQueueState() << endl;
-
- //6. getCluster
- list<NodeState> nodeStates;
- nodeStates.push_back(NodeState::NS_RUNNING);
- list<NodeReport> nodeReports = client.getClusterNodes(nodeStates);
- for (list<NodeReport>::iterator it = nodeReports.begin(); it != nodeReports.end(); it++) {
- cout << "host:" << it->getNodeId().getHost() << ", port:"
- << it->getNodeId().getPort() << ", httpAddress:" << it->getHttpAddress()
- << ", rackName:" << it->getRackName() << ", used:[vcore:"
- << it->getUsedResource().getVirtualCores() << ", mem:"
- << it->getUsedResource().getMemory() << "], capability:[vcore:"
- << it->getResourceCapability().getVirtualCores() << ", mem:"
- << it->getResourceCapability().getMemory() << "], numContainers:"
- << it->getNumContainers() << ", node_state:" << it->getNodeState()
- << ", health_report:" << it->getHealthReport()
- << ", last_health_report_time:" << it->getLastHealthReportTime()
- << endl;
- }
-
- //7. finish
- cout << "to finish: jobId:" << jobId << endl;
- client.finishJob(jobId, FinalApplicationStatus::APP_SUCCEEDED);
-
- sleep(1);
-
-
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/06b7f2f0/depends/libyarn/ztest_c/Makefile
----------------------------------------------------------------------
diff --git a/depends/libyarn/ztest_c/Makefile b/depends/libyarn/ztest_c/Makefile
deleted file mode 100644
index bfd7a61..0000000
--- a/depends/libyarn/ztest_c/Makefile
+++ /dev/null
@@ -1,10 +0,0 @@
-CC=gcc
-CPPFLAGS=-g -Wall -std=gnu99 -I../dist/include
-LDFLAGS=-L../dist/lib -L/Users/yjin/greenplum-db-devel/lib -lyarn
-#bin = c_client_main_profile
-bin = c_client_main
-
-all: $(bin)
-
-clean:
- rm -rf $(bin) *.dSYM
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/06b7f2f0/depends/libyarn/ztest_c/c_client_main.c
----------------------------------------------------------------------
diff --git a/depends/libyarn/ztest_c/c_client_main.c b/depends/libyarn/ztest_c/c_client_main.c
deleted file mode 100644
index bf00dab..0000000
--- a/depends/libyarn/ztest_c/c_client_main.c
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include <unistd.h>
-#include <stdio.h>
-#include <sys/time.h>
-#include "libyarn/LibYarnClientC.h"
-#include "libyarn/LibYarnConstants.h"
-
-int main() {
- char *rmHost = "master";
- char *rmPort = "8032";
- char *schedHost = "master";
- char *schedPort = "8030";
- char *amHost = "master";
- int32_t amPort = 8090;
- char *am_tracking_url = "url";
- int heartbeatInterval = 1000000;
- int i=0;
-
- //0. new client
- LibYarnClient_t *client = NULL;
- int result = newLibYarnClient(rmHost, rmPort, schedHost, schedPort,
- amHost, amPort, am_tracking_url,&client,heartbeatInterval);
- printf("newLibYarnClient Result Code:%d\n",result);
-
- //1. createJob
- char *jobName = "libyarn";
- char *queue = "sample_queue";
- char *jobId = NULL;
- result = createJob(client, jobName, queue,&jobId);
- printf("1. createJob, jobid:%s The createJob Result Code:%d\n", jobId,result);
- if (result != FUNCTION_SUCCEEDED){
- const char* errorMessage = getErrorMessage();
- printf("1. createJob, errorMessage:%s\n",errorMessage);
- }
-
-
- char *blackListAdditions[0];
- char *blackListRemovals[0];
-
- //1. allocate resource
- LibYarnResource_t *allocatedResourcesArray;
- int allocatedResourceArraySize;
- result = allocateResources(client, jobId, 1, 1, 1024, 5,
- blackListAdditions, 0, blackListRemovals, 0, NULL, 0,
- &allocatedResourcesArray, &allocatedResourceArraySize);
- printf("The allocateResources Result Code:%d\n",result);
- if (result != FUNCTION_SUCCEEDED){
- const char* errorMessage = getErrorMessage();
- printf("2. allocateResources, errorMessage:%s\n",errorMessage);
- }
-
- int32_t activeContainerIds[allocatedResourceArraySize];
- int32_t releaseContainerIds[allocatedResourceArraySize];
- int32_t statusContainerIds[allocatedResourceArraySize];
- printf("2. allocateResources, allocatedResourceArraySize:%d\n", allocatedResourceArraySize);
- for (i = 0 ; i < allocatedResourceArraySize; i++) {
- puts("----------------------------");
- printf("allocatedResourcesArray[i].containerId:%d\n", allocatedResourcesArray[i].containerId);
- activeContainerIds[i] = allocatedResourcesArray[i].containerId;
- releaseContainerIds[i] = allocatedResourcesArray[i].containerId;
- statusContainerIds[i] = allocatedResourcesArray[i].containerId;
- printf("allocatedResourcesArray[i].host:%s\n", allocatedResourcesArray[i].host);
- printf("allocatedResourcesArray[i].port:%d\n", allocatedResourcesArray[i].port);
- printf("allocatedResourcesArray[i].nodeHttpAddress:%s\n", allocatedResourcesArray[i].nodeHttpAddress);
- printf("allocatedResourcesArray[i].vCores:%d\n", allocatedResourcesArray[i].vCores);
- printf("allocatedResourcesArray[i].memory:%d\n", allocatedResourcesArray[i].memory);
- }
-
- //3. active
- result = activeResources(client, jobId, activeContainerIds,allocatedResourceArraySize);
- printf("The activeResources Result Code:%d\n",result);
- if (result != FUNCTION_SUCCEEDED){
- const char* errorMessage = getErrorMessage();
- printf("2. activeResources, errorMessage:%s\n",errorMessage);
- }
-
- sleep(10);
-
- int *activeFailIds;
- int activeFailSize;
- result = getActiveFailContainerIds(client,&activeFailIds,&activeFailSize);
- printf("Active Fail Container Size:%d\n",activeFailSize);
- for (i = 0;i < activeFailSize;i++){
- printf("Active Fail Container Id:%d\n",activeFailIds[i]);
- }
-
- //4. getContainerReport
- LibYarnContainerReport_t *containerReportArray;
- int containerReportArraySize;
- result = getContainerReports(client, jobId, &containerReportArray,
- &containerReportArraySize);
- printf("The getContainerReports Result Code:%d\n", result);
- if (result != FUNCTION_SUCCEEDED) {
- const char* errorMessage = getErrorMessage();
- printf(" The getContainerReports, errorMessage:%s\n", errorMessage);
- }
-
- printf("containerReportArraySize=%d\n", containerReportArraySize);
- for (i = 0; i < containerReportArraySize; i++) {
- printf("-------------container: %d--------------------------\n", i);
- printf("containerId:%d\n", containerReportArray[i].containerId);
- printf("vCores:%d\n", containerReportArray[i].vCores);
- printf("memory:%d\n", containerReportArray[i].memory);
- printf("host:%s\n", containerReportArray[i].host);
- printf("port:%d\n", containerReportArray[i].port);
- printf("exitStatus:%d\n", containerReportArray[i].exitStatus);
- printf("state:%d\n", containerReportArray[i].state);
- }
- freeContainerReportArray(containerReportArray, containerReportArraySize);
-
- //4. getContainerReport
- LibYarnContainerStatus_t *containerStatusArray;
- int containerStatusArraySize;
- result = getContainerStatuses(client, jobId, statusContainerIds,
- allocatedResourceArraySize, &containerStatusArray,
- &containerStatusArraySize);
- printf("The getContainerStatus Result Code:%d\n", result);
- if (result != FUNCTION_SUCCEEDED) {
- const char* errorMessage = getErrorMessage();
- printf(" The getContainerStatus, errorMessage:%s\n", errorMessage);
- }
-
- printf("containerStatusArraySize=%d\n", containerStatusArraySize);
- for (i = 0; i < containerStatusArraySize; i++) {
- printf("-------------container: %d--------------------------\n", i);
- printf("containerId:%d\n", containerStatusArray[i].containerId);
- printf("exitStatus:%d\n", containerStatusArray[i].exitStatus);
- printf("state:%d\n", containerStatusArray[i].state);
- printf("diagnostics:%s\n", containerStatusArray[i].diagnostics);
- }
- freeContainerStatusArray(containerStatusArray,containerStatusArraySize);
-
- //6. getQueueInfo
- LibYarnQueueInfo_t *queueInfo = NULL;
- result = getQueueInfo(client, queue, true, true, true, &queueInfo);
- printf("The getQueueInfo Result Code:%d\n",result);
- if (result != FUNCTION_SUCCEEDED){
- const char* errorMessage = getErrorMessage();
- printf(" The getQueueInfo, errorMessage:%s\n",errorMessage);
- }
-
- printf("QueueInfo: queueInfo->queueName:%s, queueInfo->capacity:%f, queueInfo->maximumCapacity:%f, queueInfo->currentCapacity:%f, queueInfo->state:%d\n",
- queueInfo->queueName, queueInfo->capacity, queueInfo->maximumCapacity,
- queueInfo->currentCapacity, queueInfo->state);
- puts("---------chilldQueue:");
- for (i = 0; i < queueInfo->childQueueNameArraySize; i++) {
- printf("QueueInfo: queueInfo->childQueueNameArray[%d]:%s\n", i, queueInfo->childQueueNameArray[i]);
- }
- freeMemQueueInfo(queueInfo);
-
- //7. getCluster
- LibYarnNodeReport_t *nodeReportArray;
- int nodeReportArraySize;
- result = getClusterNodes(client, NODE_STATE_RUNNING, &nodeReportArray, &nodeReportArraySize);
- printf("The getClusterNodes Result Code:%d\n",result);
- if (result != FUNCTION_SUCCEEDED){
- const char* errorMessage = getErrorMessage();
- printf(" The getClusterNodes, errorMessage:%s\n",errorMessage);
- }
-
- printf("nodeReportArraySize=%d\n", nodeReportArraySize);
- for (i = 0; i < nodeReportArraySize; i++) {
- printf("-------------node %d--------------------------\n", i);
- printf("host:%s\n", nodeReportArray[i].host);
- printf("port:%d\n", nodeReportArray[i].port);
- printf("httpAddress:%s\n", nodeReportArray[i].httpAddress);
- printf("rackName:%s\n", nodeReportArray[i].rackName);
- printf("memoryUsed:%d\n", nodeReportArray[i].memoryUsed);
- printf("vcoresUsed:%d\n", nodeReportArray[i].vcoresUsed);
- printf("memoryCapability:%d\n", nodeReportArray[i].memoryCapability);
- printf("vcoresCapability:%d\n", nodeReportArray[i].vcoresCapability);
- printf("numContainers:%d\n", nodeReportArray[i].numContainers);
- printf("nodeState:%d\n", nodeReportArray[i].nodeState);
- printf("healthReport:%s\n", nodeReportArray[i].healthReport);
- printf("lastHealthReportTime:%lld\n", nodeReportArray[i].lastHealthReportTime);
- }
- freeMemNodeReportArray(nodeReportArray, nodeReportArraySize);
-
- //8. getApplicationReport
- LibYarnApplicationReport_t *applicationReport = NULL;
- result = getApplicationReport(client, jobId, &applicationReport);
- printf("The getApplicationReport Result Code:%d\n", result);
- if (result != FUNCTION_SUCCEEDED) {
- const char* errorMessage = getErrorMessage();
- printf(" The getApplicationReport, errorMessage:%s\n",
- errorMessage);
- }
- printf("-------------ApplicationReport-------------------------\n");
- printf("appId:%d\n", applicationReport->appId);
- printf("user:%s\n", applicationReport->user);
- printf("queue:%s\n", applicationReport->queue);
- printf("name:%s\n", applicationReport->name);
- printf("host:%s\n", applicationReport->host);
- printf("port:%d\n", applicationReport->port);
- printf("status:%d\n", applicationReport->status);
-
- freeApplicationReport(applicationReport);
-
- //5. release
- result = releaseResources(client, jobId, releaseContainerIds,allocatedResourceArraySize);
- printf("The releaseResources Result Code:%d\n",result);
-
- result = getActiveFailContainerIds(client,&activeFailIds,&activeFailSize);
- printf("Active Fail Container Size:%d\n",activeFailSize);
- for (i = 0;i < activeFailSize;i++){
- printf("Active Fail Container Id:%d\n",activeFailIds[i]);
- }
- free(activeFailIds);
- freeMemAllocatedResourcesArray(allocatedResourcesArray, allocatedResourceArraySize);
- printf("freeMemAllocated is OK\n");
-
- //9. finish
- printf("jobId:%s\n", jobId);
- result = finishJob(client, jobId, APPLICATION_SUCCEEDED);
- printf("The finishJob Result Code:%d\n", result);
- if (result != FUNCTION_SUCCEEDED) {
- const char* errorMessage = getErrorMessage();
- printf(" The finishJob, errorMessage:%s\n", errorMessage);
- }
- free(jobId);
- //10. free client
- deleteLibYarnClient(client);
-
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/06b7f2f0/depends/libyarn/ztest_c/c_client_main_profile.c
----------------------------------------------------------------------
diff --git a/depends/libyarn/ztest_c/c_client_main_profile.c b/depends/libyarn/ztest_c/c_client_main_profile.c
deleted file mode 100644
index b94c919..0000000
--- a/depends/libyarn/ztest_c/c_client_main_profile.c
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#include <unistd.h>
-#include <stdio.h>
-#include <sys/time.h>
-#include "libyarn/LibYarnClientC.h"
-#include "libyarn/LibYarnConstants.h"
-
-int main() {
- struct timeval tpstart,tpend;
- float allocateTime = 0;
- float activeTime = 0;
- float releaseTime = 0;
- float timeuse;
- char *rmHost = "localhost";
- char *rmPort = "8032";
- char *schedHost = "localhost";
- char *schedPort = "8030";
- char *amHost = "localhost";
- int32_t amPort = 10;
- char *am_tracking_url = "url";
- int heartbeatInterval = 1000;
-
- //0. new client
- LibYarnClient_t *client = NULL;
- int result = newLibYarnClient(rmHost, rmPort, schedHost, schedPort,
- amHost, amPort, am_tracking_url,&client,heartbeatInterval);
-
- //1. createJob
- char *jobName = "libyarn";
- char *queue = "default";
- char *jobId = NULL;
- result = createJob(client, jobName, queue,&jobId);
- if (result != FUNCTION_SUCCEEDED){
- const char* errorMessage = getErrorMessage();
- printf("1. createJob, errorMessage:%s\n",errorMessage);
- }
-
- int i,j;
- for (j = 0;j < 10;j++){
- LibYarnResourceRequest_t resRequest;
- resRequest.priority = 1;
- resRequest.host = "*";
- resRequest.vCores = 1;
- resRequest.memory = 1024;
- resRequest.num_containers = 2;
- resRequest.relax_locality = 1;
-
- char *blackListAdditions[0];
- char *blackListRemovals[0];
-
- //1. allocate resource
- LibYarnResource_t *allocatedResourcesArray;
- int allocatedResourceArraySize;
- gettimeofday(&tpstart,NULL);
- result = allocateResources(client, jobId, &resRequest, blackListAdditions,
- 0, blackListRemovals, 0, &allocatedResourcesArray, &allocatedResourceArraySize,5);
- gettimeofday(&tpend,NULL);
- timeuse=1000000*(tpend.tv_sec-tpstart.tv_sec)+tpend.tv_usec-tpstart.tv_usec;
- allocateTime += timeuse;
-
- if (result != FUNCTION_SUCCEEDED){
- const char* errorMessage = getErrorMessage();
- printf("2. allocateResources, errorMessage:%s\n",errorMessage);
- }
-
- int32_t activeContainerIds[allocatedResourceArraySize];
- int32_t releaseContainerIds[allocatedResourceArraySize];
- int32_t statusContainerIds[allocatedResourceArraySize];
- for (i = 0 ; i < allocatedResourceArraySize; i++) {
- activeContainerIds[i] = allocatedResourcesArray[i].containerId;
- releaseContainerIds[i] = allocatedResourcesArray[i].containerId;
- statusContainerIds[i] = allocatedResourcesArray[i].containerId;
- }
-
- //3. active
- gettimeofday(&tpstart,NULL);
- result = activeResources(client, jobId, activeContainerIds,allocatedResourceArraySize);
- gettimeofday(&tpend,NULL);
- timeuse=1000000*(tpend.tv_sec-tpstart.tv_sec)+tpend.tv_usec-tpstart.tv_usec;
- activeTime += timeuse;
- if (result != FUNCTION_SUCCEEDED){
- const char* errorMessage = getErrorMessage();
- printf("2. activeResources, errorMessage:%s\n",errorMessage);
- }
-
- int *activeFailIds;
- int activeFailSize;
- result = getActiveFailContainerIds(client,&activeFailIds,&activeFailSize);
- for (i = 0;i < activeFailSize;i++){
- printf("Active Fail Container Id:%d\n",activeFailIds[i]);
- }
- //sleep(160);
-
- //4. getContainerReport
- LibYarnContainerReport_t *containerReportArray;
- int containerReportArraySize;
- result = getContainerReports(client, jobId, &containerReportArray,
- &containerReportArraySize);
- if (result != FUNCTION_SUCCEEDED) {
- const char* errorMessage = getErrorMessage();
- printf(" The getContainerReports, errorMessage:%s\n", errorMessage);
- }
-
- freeContainerReportArray(containerReportArray, containerReportArraySize);
-
- //4. getContainerReport
- LibYarnContainerStatus_t *containerStatusArray;
- int containerStatusArraySize;
- result = getContainerStatuses(client, jobId, statusContainerIds,
- allocatedResourceArraySize, &containerStatusArray,
- &containerStatusArraySize);
- if (result != FUNCTION_SUCCEEDED) {
- const char* errorMessage = getErrorMessage();
- printf(" The getContainerStatus, errorMessage:%s\n", errorMessage);
- }
- freeContainerStatusArray(containerStatusArray,containerStatusArraySize);
-
- //5. release
- gettimeofday(&tpstart,NULL);
- result = releaseResources(client, jobId, releaseContainerIds,allocatedResourceArraySize);
- gettimeofday(&tpend,NULL);
- timeuse=1000000*(tpend.tv_sec-tpstart.tv_sec)+tpend.tv_usec-tpstart.tv_usec;
- releaseTime += timeuse;
-
- result = getActiveFailContainerIds(client,&activeFailIds,&activeFailSize);
- for (i = 0;i < activeFailSize;i++){
- printf("Active Fail Container Id:%d\n",activeFailIds[i]);
- }
- free(activeFailIds);
- freeMemAllocatedResourcesArray(allocatedResourcesArray, allocatedResourceArraySize);
-
- //6. getQueueInfo
- LibYarnQueueInfo_t *queueInfo = NULL;
- result = getQueueInfo(client, queue, true, true, true, &queueInfo);
- if (result != FUNCTION_SUCCEEDED){
- const char* errorMessage = getErrorMessage();
- printf(" The getQueueInfo, errorMessage:%s\n",errorMessage);
- }
-
- freeMemQueueInfo(queueInfo);
-
- //7. getCluster
- LibYarnNodeReport_t *nodeReportArray;
- int nodeReportArraySize;
- result = getClusterNodes(client, NODE_STATE_RUNNING, &nodeReportArray, &nodeReportArraySize);
- if (result != FUNCTION_SUCCEEDED){
- const char* errorMessage = getErrorMessage();
- printf(" The getClusterNodes, errorMessage:%s\n",errorMessage);
- }
-
- freeMemNodeReportArray(nodeReportArray, nodeReportArraySize);
-
- //8. getApplicationReport
- LibYarnApplicationReport_t *applicationReport = NULL;
- result = getApplicationReport(client, jobId, &applicationReport);
- if (result != FUNCTION_SUCCEEDED) {
- const char* errorMessage = getErrorMessage();
- printf(" The getApplicationReport, errorMessage:%s\n",
- errorMessage);
- }
-
- freeApplicationReport(applicationReport);
- }
- printf("allocateTime:%f s\n",allocateTime/1000000);
- printf("activeTime:%f s\n",activeTime/1000000);
- printf("releaseTime:%f s\n",releaseTime/1000000);
- //9. finish
- result = finishJob(client, jobId, APPLICATION_SUCCEEDED);
- if (result != FUNCTION_SUCCEEDED) {
- const char* errorMessage = getErrorMessage();
- printf(" The finishJob, errorMessage:%s\n", errorMessage);
- }
- free(jobId);
- //10. free client
- deleteLibYarnClient(client);
-
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/06b7f2f0/src/backend/utils/misc/etc/yarn-client.xml
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/etc/yarn-client.xml b/src/backend/utils/misc/etc/yarn-client.xml
index 6a5be71..0121e74 100644
--- a/src/backend/utils/misc/etc/yarn-client.xml
+++ b/src/backend/utils/misc/etc/yarn-client.xml
@@ -112,175 +112,11 @@ HA -->
</description>
</property>
- <!-- dfs client configuration -->
<property>
- <name>dfs.client.read.shortcircuit</name>
- <value>true</value>
- <description>
- whether reading block file bypass datanode if the block and the client are on the same node. default is true.
- </description>
- </property>
-
- <property>
- <name>dfs.default.replica</name>
- <value>3</value>
- <description>
- the default number of replica. default is 3.
- </description>
- </property>
-
- <property>
- <name>dfs.prefetchsize</name>
- <value>10</value>
- <description>
- the default number of blocks which information will be prefetched. default is 10.
- </description>
- </property>
-
- <property>
- <name>dfs.client.failover.max.attempts</name>
+ <name>yarn.client.failover.max.attempts</name>
<value>15</value>
<description>
- if multiply namenodes are configured, it is the max retry times when the dfs client try to issue a RPC call. default is 15.
- </description>
- </property>
-
- <property>
- <name>dfs.default.blocksize</name>
- <value>134217728</value>
- <description>
- default block size. default is 134217728.
- </description>
- </property>
-
- <property>
- <name>dfs.client.log.severity</name>
- <value>INFO</value>
- <description>
- the minimal log severity level, valid values include FATAL, ERROR, INFO, DEBUG1, DEBUG2, DEBUG3. default is INFO.
- </description>
- </property>
-
- <!-- input client configuration -->
- <property>
- <name>input.connect.timeout</name>
- <value>600000</value>
- <description>
- the timeout interval in millisecond when the input stream is trying to setup the connection to datanode. default is 600000.
- </description>
- </property>
-
- <property>
- <name>input.read.timeout</name>
- <value>3600000</value>
- <description>
- the timeout interval in millisecond when the input stream is trying to read from datanode. default is 3600000.
- </description>
- </property>
-
- <property>
- <name>input.write.timeout</name>
- <value>3600000</value>
- <description>
- the timeout interval in millisecond when the input stream is trying to write to datanode. default is 3600000.
- </description>
- </property>
-
- <property>
- <name>input.localread.default.buffersize</name>
- <value>2097152</value>
- <description>
- number of bytes of the buffer which is used to hold the data from block file and verify checksum.
- it is only used when "dfs.client.read.shortcircuit" is set to true. default is 1048576.
- </description>
- </property>
-
- <property>
- <name>input.localread.blockinfo.cachesize</name>
- <value>1000</value>
- <description>
- the size of block file path information cache. default is 1000.
- </description>
- </property>
-
- <property>
- <name>input.read.getblockinfo.retry</name>
- <value>3</value>
- <description>
- the max retry times when the client fail to get block information from namenode. default is 3.
- </description>
- </property>
-
- <!-- output client configuration -->
- <property>
- <name>output.replace-datanode-on-failure</name>
- <value>false</value>
- <description>
- whether the client add new datanode into pipeline if the number of nodes in pipeline is less the specified number of replicas. default is true.
- </description>
- </property>
-
- <property>
- <name>output.default.chunksize</name>
- <value>512</value>
- <description>
- the number of bytes of a chunk in pipeline. default is 512.
- </description>
- </property>
-
- <property>
- <name>output.default.packetsize</name>
- <value>65536</value>
- <description>
- the number of bytes of a packet in pipeline. default is 65536.
- </description>
- </property>
-
- <property>
- <name>output.default.write.retry</name>
- <value>10</value>
- <description>
- the max retry times when the client fail to setup the pipeline. default is 10.
- </description>
- </property>
-
- <property>
- <name>output.connect.timeout</name>
- <value>600000</value>
- <description>
- the timeout interval in millisecond when the output stream is trying to setup the connection to datanode. default is 600000.
- </description>
- </property>
-
- <property>
- <name>output.read.timeout</name>
- <value>3600000</value>
- <description>
- the timeout interval in millisecond when the output stream is trying to read from datanode. default is 3600000.
- </description>
- </property>
-
- <property>
- <name>output.write.timeout</name>
- <value>3600000</value>
- <description>
- the timeout interval in millisecond when the output stream is trying to write to datanode. default is 3600000.
- </description>
- </property>
-
- <property>
- <name>output.packetpool.size</name>
- <value>1024</value>
- <description>
- the max number of packets in a file's packet pool. default is 1024.
- </description>
- </property>
-
- <property>
- <name>output.close.timeout</name>
- <value>900000</value>
- <description>
- the timeout interval in millisecond when close an output stream. default is 900000.
+ if multiply resource managers are configured, it is the max retry times when the yarn client try to issue a RPC call. default is 15.
</description>
</property>
</configuration>