You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by su...@apache.org on 2013/01/22 20:33:25 UTC
svn commit: r1437113 [5/5] - in
/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project: ./ conf/
dev-support/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduc...
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java Tue Jan 22 19:33:02 2013
@@ -1,3 +1,20 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
package org.apache.hadoop.mapreduce.v2.hs;
import java.io.IOException;
@@ -6,6 +23,7 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.TaskCompletionEvent;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
@@ -126,7 +144,7 @@ public class MockHistoryJobs extends Moc
}
@Override
- public TaskAttemptCompletionEvent[] getMapAttemptCompletionEvents(
+ public TaskCompletionEvent[] getMapAttemptCompletionEvents(
int startIndex, int maxEvents) {
return job.getMapAttemptCompletionEvents(startIndex, maxEvents);
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java Tue Jan 22 19:33:02 2013
@@ -1,3 +1,20 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
package org.apache.hadoop.mapreduce.v2.hs;
import static junit.framework.Assert.assertEquals;
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml Tue Jan 22 19:33:02 2013
@@ -148,6 +148,15 @@
</additionalClasspathElements>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>src/test/java/org/apache/hadoop/cli/data60bytes</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
</plugins>
</build>
</project>
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java Tue Jan 22 19:33:02 2013
@@ -33,6 +33,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -69,14 +70,16 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ClientToken;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier;
+import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier;
import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.hadoop.yarn.util.ProtoUtils;
public class ClientServiceDelegate {
private static final Log LOG = LogFactory.getLog(ClientServiceDelegate.class);
@@ -176,13 +179,10 @@ public class ClientServiceDelegate {
serviceAddr = NetUtils.createSocketAddrForHost(
application.getHost(), application.getRpcPort());
if (UserGroupInformation.isSecurityEnabled()) {
- String clientTokenEncoded = application.getClientToken();
- Token<ApplicationTokenIdentifier> clientToken =
- new Token<ApplicationTokenIdentifier>();
- clientToken.decodeFromUrlString(clientTokenEncoded);
- // RPC layer client expects ip:port as service for tokens
- SecurityUtil.setTokenService(clientToken, serviceAddr);
- newUgi.addToken(clientToken);
+ ClientToken clientToken = application.getClientToken();
+ Token<ClientTokenIdentifier> token =
+ ProtoUtils.convertFromProtoFormat(clientToken, serviceAddr);
+ newUgi.addToken(token);
}
LOG.debug("Connecting to " + serviceAddr);
final InetSocketAddress finalServiceAddr = serviceAddr;
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java Tue Jan 22 19:33:02 2013
@@ -24,6 +24,8 @@ import java.util.HashMap;
import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest;
@@ -48,6 +50,8 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse;
import org.apache.hadoop.mapreduce.v2.api.records.CounterGroup;
import org.apache.hadoop.mapreduce.v2.api.records.Counters;
import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
@@ -58,8 +62,8 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -82,10 +86,9 @@ public class NotRunningJob implements MR
.newRecordInstance(ApplicationAttemptId.class);
// Setting AppState to NEW and finalStatus to UNDEFINED as they are never
- // used
- // for a non running job
+ // used for a non running job
return BuilderUtils.newApplicationReport(unknownAppId, unknownAttemptId,
- "N/A", "N/A", "N/A", "N/A", 0, "", YarnApplicationState.NEW, "N/A",
+ "N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A",
"N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A");
}
@@ -215,7 +218,21 @@ public class NotRunningJob implements MR
/* Should not be invoked by anyone. */
throw new NotImplementedException();
}
-
+
+ @Override
+ public RenewDelegationTokenResponse renewDelegationToken(
+ RenewDelegationTokenRequest request) throws YarnRemoteException {
+ /* Should not be invoked by anyone. */
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public CancelDelegationTokenResponse cancelDelegationToken(
+ CancelDelegationTokenRequest request) throws YarnRemoteException {
+ /* Should not be invoked by anyone. */
+ throw new NotImplementedException();
+ }
+
@Override
public InetSocketAddress getConnectAddress() {
/* Should not be invoked by anyone. Normally used to set token service */
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java Tue Jan 22 19:33:02 2013
@@ -33,7 +33,6 @@ import org.apache.hadoop.mapreduce.Queue
import org.apache.hadoop.mapreduce.QueueInfo;
import org.apache.hadoop.mapreduce.TaskTrackerInfo;
import org.apache.hadoop.mapreduce.TypeConverter;
-import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -41,9 +40,9 @@ import org.apache.hadoop.yarn.api.Client
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
+import org.apache.hadoop.yarn.client.YarnClientImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.ProtoUtils;
-import org.apache.hadoop.yarn.client.YarnClientImpl;
public class ResourceMgrDelegate extends YarnClientImpl {
private static final Log LOG = LogFactory.getLog(ResourceMgrDelegate.class);
@@ -62,11 +61,6 @@ public class ResourceMgrDelegate extends
init(conf);
start();
}
-
- public void cancelDelegationToken(Token<DelegationTokenIdentifier> arg0)
- throws IOException, InterruptedException {
- return;
- }
public TaskTrackerInfo[] getActiveTrackers() throws IOException,
InterruptedException {
@@ -168,13 +162,6 @@ public class ResourceMgrDelegate extends
return 0;
}
- public long renewDelegationToken(Token<DelegationTokenIdentifier> arg0)
- throws IOException, InterruptedException {
- // TODO: Implement renewDelegationToken
- LOG.warn("renewDelegationToken - Not implemented");
- return 0;
- }
-
public ApplicationId getApplicationId() {
return applicationId;
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Tue Jan 22 19:33:02 2013
@@ -85,6 +85,7 @@ import org.apache.hadoop.yarn.util.Build
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.ProtoUtils;
+import com.google.common.annotations.VisibleForTesting;
/**
* This class enables the current JobClient (0.22 hadoop) to run on YARN.
@@ -100,7 +101,7 @@ public class YARNRunner implements Clien
private Configuration conf;
private final FileContext defaultFileContext;
- /* usually is false unless the jobclient getdelegation token is
+ /* usually is false unless the jobclient get delegation token is
* called. This is a hack wherein we do return a token from RM
* on getDelegationtoken but due to the restricted api on jobclient
* we just add a job history DT token when submitting a job.
@@ -158,18 +159,18 @@ public class YARNRunner implements Clien
@Override
public void cancelDelegationToken(Token<DelegationTokenIdentifier> arg0)
throws IOException, InterruptedException {
- resMgrDelegate.cancelDelegationToken(arg0);
+ throw new UnsupportedOperationException("Use Token.renew instead");
}
@Override
public TaskTrackerInfo[] getActiveTrackers() throws IOException,
InterruptedException {
- return resMgrDelegate.getActiveTrackers();
+ return resMgrDelegate.getActiveTrackers();
}
@Override
public JobStatus[] getAllJobs() throws IOException, InterruptedException {
- return resMgrDelegate.getAllJobs();
+ return resMgrDelegate.getAllJobs();
}
@Override
@@ -184,18 +185,18 @@ public class YARNRunner implements Clien
return resMgrDelegate.getClusterMetrics();
}
- private Token<?> getDelegationTokenFromHS(
- MRClientProtocol hsProxy, Text renewer) throws IOException,
- InterruptedException {
+ @VisibleForTesting
+ Token<?> getDelegationTokenFromHS(MRClientProtocol hsProxy)
+ throws IOException, InterruptedException {
GetDelegationTokenRequest request = recordFactory
.newRecordInstance(GetDelegationTokenRequest.class);
- request.setRenewer(renewer.toString());
+ request.setRenewer(Master.getMasterPrincipal(conf));
DelegationToken mrDelegationToken = hsProxy.getDelegationToken(request)
.getDelegationToken();
return ProtoUtils.convertFromProtoFormat(mrDelegationToken,
hsProxy.getConnectAddress());
}
-
+
@Override
public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
throws IOException, InterruptedException {
@@ -269,8 +270,7 @@ public class YARNRunner implements Clien
// the delegation tokens for the HistoryServer also.
if (conf.getBoolean(JobClient.HS_DELEGATION_TOKEN_REQUIRED,
DEFAULT_HS_DELEGATION_TOKEN_REQUIRED)) {
- Token hsDT = getDelegationTokenFromHS(hsProxy, new Text(
- conf.get(JobClient.HS_DELEGATION_TOKEN_RENEWER)));
+ Token hsDT = getDelegationTokenFromHS(hsProxy);
ts.addToken(hsDT.getService(), hsDT);
}
}
@@ -324,8 +324,16 @@ public class YARNRunner implements Clien
// Setup resource requirements
Resource capability = recordFactory.newRecordInstance(Resource.class);
- capability.setMemory(conf.getInt(MRJobConfig.MR_AM_VMEM_MB,
- MRJobConfig.DEFAULT_MR_AM_VMEM_MB));
+ capability.setMemory(
+ conf.getInt(
+ MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB
+ )
+ );
+ capability.setVirtualCores(
+ conf.getInt(
+ MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES
+ )
+ );
LOG.debug("AppMaster capability = " + capability);
// Setup LocalResources
@@ -386,9 +394,31 @@ public class YARNRunner implements Clien
MRJobConfig.MR_AM_LOG_LEVEL, MRJobConfig.DEFAULT_MR_AM_LOG_LEVEL);
MRApps.addLog4jSystemProperties(logLevel, logSize, vargs);
- vargs.add(conf.get(MRJobConfig.MR_AM_COMMAND_OPTS,
- MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS));
-
+ // Check for Java Lib Path usage in MAP and REDUCE configs
+ warnForJavaLibPath(conf.get(MRJobConfig.MAP_JAVA_OPTS,""), "map",
+ MRJobConfig.MAP_JAVA_OPTS, MRJobConfig.MAP_ENV);
+ warnForJavaLibPath(conf.get(MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS,""), "map",
+ MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV);
+ warnForJavaLibPath(conf.get(MRJobConfig.REDUCE_JAVA_OPTS,""), "reduce",
+ MRJobConfig.REDUCE_JAVA_OPTS, MRJobConfig.REDUCE_ENV);
+ warnForJavaLibPath(conf.get(MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS,""), "reduce",
+ MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV);
+
+ // Add AM admin command opts before user command opts
+ // so that it can be overridden by user
+ String mrAppMasterAdminOptions = conf.get(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS,
+ MRJobConfig.DEFAULT_MR_AM_ADMIN_COMMAND_OPTS);
+ warnForJavaLibPath(mrAppMasterAdminOptions, "app master",
+ MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, MRJobConfig.MR_AM_ADMIN_USER_ENV);
+ vargs.add(mrAppMasterAdminOptions);
+
+ // Add AM user command opts
+ String mrAppMasterUserOptions = conf.get(MRJobConfig.MR_AM_COMMAND_OPTS,
+ MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS);
+ warnForJavaLibPath(mrAppMasterUserOptions, "app master",
+ MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.MR_AM_ENV);
+ vargs.add(mrAppMasterUserOptions);
+
vargs.add(MRJobConfig.APPLICATION_MASTER_CLASS);
vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
Path.SEPARATOR + ApplicationConstants.STDOUT);
@@ -412,6 +442,9 @@ public class YARNRunner implements Clien
Map<String, String> environment = new HashMap<String, String>();
MRApps.setClasspath(environment, conf);
+ // Setup the environment variables for Admin first
+ MRApps.setEnvFromInputString(environment,
+ conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV));
// Setup the environment variables (LD_LIBRARY_PATH, etc)
MRApps.setEnvFromInputString(environment,
conf.get(MRJobConfig.MR_AM_ENV));
@@ -466,7 +499,7 @@ public class YARNRunner implements Clien
@Override
public long renewDelegationToken(Token<DelegationTokenIdentifier> arg0)
throws IOException, InterruptedException {
- return resMgrDelegate.renewDelegationToken(arg0);
+ throw new UnsupportedOperationException("Use Token.renew instead");
}
@@ -569,4 +602,15 @@ public class YARNRunner implements Clien
throws IOException {
return clientCache.getClient(jobID).getLogFilePath(jobID, taskAttemptID);
}
+
+ private static void warnForJavaLibPath(String opts, String component,
+ String javaConf, String envConf) {
+ if (opts != null && opts.contains("-Djava.library.path")) {
+ LOG.warn("Usage of -Djava.library.path in " + javaConf + " can cause " +
+ "programs to no longer function if hadoop native libraries " +
+ "are used. These values should be set as part of the " +
+ "LD_LIBRARY_PATH in the " + component + " JVM env using " +
+ envConf + " config settings.");
+ }
+ }
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo Tue Jan 22 19:33:02 2013
@@ -1 +1,14 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
org.apache.hadoop.mapreduce.v2.security.client.ClientHSSecurityInfo
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java Tue Jan 22 19:33:02 2013
@@ -90,8 +90,8 @@ public abstract class NotificationTestCa
}
public static class NotificationServlet extends HttpServlet {
- public static int counter = 0;
- public static int failureCounter = 0;
+ public static volatile int counter = 0;
+ public static volatile int failureCounter = 0;
private static final long serialVersionUID = 1L;
protected void doGet(HttpServletRequest req, HttpServletResponse res)
@@ -155,7 +155,11 @@ public abstract class NotificationTestCa
System.out.println(launchWordCount(this.createJobConf(),
"a b c d e f g h", 1, 1));
- Thread.sleep(2000);
+ boolean keepTrying = true;
+ for (int tries = 0; tries < 30 && keepTrying; tries++) {
+ Thread.sleep(50);
+ keepTrying = !(NotificationServlet.counter == 2);
+ }
assertEquals(2, NotificationServlet.counter);
assertEquals(0, NotificationServlet.failureCounter);
@@ -173,14 +177,22 @@ public abstract class NotificationTestCa
// run a job with KILLED status
System.out.println(UtilsForTests.runJobKill(this.createJobConf(), inDir,
outDir).getID());
- Thread.sleep(2000);
+ keepTrying = true;
+ for (int tries = 0; tries < 30 && keepTrying; tries++) {
+ Thread.sleep(50);
+ keepTrying = !(NotificationServlet.counter == 4);
+ }
assertEquals(4, NotificationServlet.counter);
assertEquals(0, NotificationServlet.failureCounter);
// run a job with FAILED status
System.out.println(UtilsForTests.runJobFail(this.createJobConf(), inDir,
outDir).getID());
- Thread.sleep(2000);
+ keepTrying = true;
+ for (int tries = 0; tries < 30 && keepTrying; tries++) {
+ Thread.sleep(50);
+ keepTrying = !(NotificationServlet.counter == 6);
+ }
assertEquals(6, NotificationServlet.counter);
assertEquals(0, NotificationServlet.failureCounter);
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java Tue Jan 22 19:33:02 2013
@@ -25,10 +25,10 @@ import java.util.Iterator;
import junit.framework.Assert;
-import org.apache.hadoop.ipc.Server;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.mapreduce.Cluster;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.MRConfig;
@@ -69,6 +69,8 @@ import org.apache.hadoop.metrics2.lib.De
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.ClientRMProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
@@ -87,6 +89,8 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -339,6 +343,18 @@ public class TestClientRedirect {
GetDelegationTokenRequest request) throws YarnRemoteException {
return null;
}
+
+ @Override
+ public RenewDelegationTokenResponse renewDelegationToken(
+ RenewDelegationTokenRequest request) throws YarnRemoteException {
+ return null;
+ }
+
+ @Override
+ public CancelDelegationTokenResponse cancelDelegationToken(
+ CancelDelegationTokenRequest request) throws YarnRemoteException {
+ return null;
+ }
}
class HistoryService extends AMService implements HSClientProtocol {
@@ -504,6 +520,20 @@ public class TestClientRedirect {
throws YarnRemoteException {
return null;
}
+
+ @Override
+ public org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse renewDelegationToken(
+ org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest request)
+ throws YarnRemoteException {
+ return null;
+ }
+
+ @Override
+ public org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse cancelDelegationToken(
+ org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest request)
+ throws YarnRemoteException {
+ return null;
+ }
}
static Counters getMyCounters() {
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java Tue Jan 22 19:33:02 2013
@@ -18,19 +18,29 @@
package org.apache.hadoop.mapreduce.security;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
import java.io.IOException;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
import java.security.PrivilegedExceptionAction;
import junit.framework.Assert;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.v2.api.HSClientProtocol;
import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest;
+import org.apache.hadoop.mapreduce.v2.hs.JHSDelegationTokenSecretManager;
import org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
@@ -49,6 +59,8 @@ import org.junit.Test;
public class TestJHSSecurity {
+ private static final Log LOG = LogFactory.getLog(TestJHSSecurity.class);
+
@Test
public void testDelegationToken() throws IOException, InterruptedException {
@@ -63,55 +75,208 @@ public class TestJHSSecurity {
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
UserGroupInformation.setConfiguration(conf);
-
- final JobHistoryServer jobHistoryServer = new JobHistoryServer() {
- protected void doSecureLogin(Configuration conf) throws IOException {
- // no keytab based login
+
+ final long initialInterval = 10000l;
+ final long maxLifetime= 20000l;
+ final long renewInterval = 10000l;
+
+ JobHistoryServer jobHistoryServer = null;
+ MRClientProtocol clientUsingDT = null;
+ long tokenFetchTime;
+ try {
+ jobHistoryServer = new JobHistoryServer() {
+ protected void doSecureLogin(Configuration conf) throws IOException {
+ // no keytab based login
+ };
+
+ protected JHSDelegationTokenSecretManager createJHSSecretManager(
+ Configuration conf) {
+ return new JHSDelegationTokenSecretManager(initialInterval,
+ maxLifetime, renewInterval, 3600000);
+ }
};
- };
- jobHistoryServer.init(conf);
- jobHistoryServer.start();
-
- // Fake the authentication-method
- UserGroupInformation loggedInUser = UserGroupInformation.getCurrentUser();
- loggedInUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS);
+// final JobHistoryServer jobHistoryServer = jhServer;
+ jobHistoryServer.init(conf);
+ jobHistoryServer.start();
+ final MRClientProtocol hsService = jobHistoryServer.getClientService()
+ .getClientHandler();
+
+ // Fake the authentication-method
+ UserGroupInformation loggedInUser = UserGroupInformation
+ .createRemoteUser("testrenewer@APACHE.ORG");
+ Assert.assertEquals("testrenewer", loggedInUser.getShortUserName());
+ // Default realm is APACHE.ORG
+ loggedInUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS);
+
+
+ DelegationToken token = getDelegationToken(loggedInUser, hsService,
+ loggedInUser.getShortUserName());
+ tokenFetchTime = System.currentTimeMillis();
+ LOG.info("Got delegation token at: " + tokenFetchTime);
+
+ // Now try talking to JHS using the delegation token
+ clientUsingDT = getMRClientProtocol(token, jobHistoryServer
+ .getClientService().getBindAddress(), "TheDarkLord", conf);
+
+ GetJobReportRequest jobReportRequest =
+ Records.newRecord(GetJobReportRequest.class);
+ jobReportRequest.setJobId(MRBuilderUtils.newJobId(123456, 1, 1));
+ try {
+ clientUsingDT.getJobReport(jobReportRequest);
+ } catch (YarnRemoteException e) {
+ Assert.assertEquals("Unknown job job_123456_0001", e.getMessage());
+ }
+
+ // Renew after 50% of token age.
+ while(System.currentTimeMillis() < tokenFetchTime + initialInterval / 2) {
+ Thread.sleep(500l);
+ }
+ long nextExpTime = renewDelegationToken(loggedInUser, hsService, token);
+ long renewalTime = System.currentTimeMillis();
+ LOG.info("Renewed token at: " + renewalTime + ", NextExpiryTime: "
+ + nextExpTime);
+
+ // Wait for first expiry, but before renewed expiry.
+ while (System.currentTimeMillis() > tokenFetchTime + initialInterval
+ && System.currentTimeMillis() < nextExpTime) {
+ Thread.sleep(500l);
+ }
+ Thread.sleep(50l);
+
+ // Valid token because of renewal.
+ try {
+ clientUsingDT.getJobReport(jobReportRequest);
+ } catch (UndeclaredThrowableException e) {
+ Assert.assertEquals("Unknown job job_123456_0001", e.getMessage());
+ }
+
+ // Wait for expiry.
+ while(System.currentTimeMillis() < renewalTime + renewInterval) {
+ Thread.sleep(500l);
+ }
+ Thread.sleep(50l);
+ LOG.info("At time: " + System.currentTimeMillis() + ", token should be invalid");
+ // Token should have expired.
+ try {
+ clientUsingDT.getJobReport(jobReportRequest);
+ fail("Should not have succeeded with an expired token");
+ } catch (UndeclaredThrowableException e) {
+ assertTrue(e.getCause().getMessage().contains("is expired"));
+ }
+
+ // Test cancellation
+ // Stop the existing proxy, start another.
+ if (clientUsingDT != null) {
+// RPC.stopProxy(clientUsingDT);
+ clientUsingDT = null;
+ }
+ token = getDelegationToken(loggedInUser, hsService,
+ loggedInUser.getShortUserName());
+ tokenFetchTime = System.currentTimeMillis();
+ LOG.info("Got delegation token at: " + tokenFetchTime);
+
+ // Now try talking to HSService using the delegation token
+ clientUsingDT = getMRClientProtocol(token, jobHistoryServer
+ .getClientService().getBindAddress(), "loginuser2", conf);
+
+
+ try {
+ clientUsingDT.getJobReport(jobReportRequest);
+ } catch (UndeclaredThrowableException e) {
+ fail("Unexpected exception" + e);
+ }
+ cancelDelegationToken(loggedInUser, hsService, token);
+ if (clientUsingDT != null) {
+// RPC.stopProxy(clientUsingDT);
+ clientUsingDT = null;
+ }
+
+ // Creating a new connection.
+ clientUsingDT = getMRClientProtocol(token, jobHistoryServer
+ .getClientService().getBindAddress(), "loginuser2", conf);
+ LOG.info("Cancelled delegation token at: " + System.currentTimeMillis());
+ // Verify cancellation worked.
+ try {
+ clientUsingDT.getJobReport(jobReportRequest);
+ fail("Should not have succeeded with a cancelled delegation token");
+ } catch (UndeclaredThrowableException e) {
+ }
+
+
+
+ } finally {
+ jobHistoryServer.stop();
+ }
+ }
+ private DelegationToken getDelegationToken(
+ final UserGroupInformation loggedInUser,
+ final MRClientProtocol hsService, final String renewerString)
+ throws IOException, InterruptedException {
// Get the delegation token directly as it is a little difficult to setup
// the kerberos based rpc.
- DelegationToken token =
- loggedInUser.doAs(new PrivilegedExceptionAction<DelegationToken>() {
+ DelegationToken token = loggedInUser
+ .doAs(new PrivilegedExceptionAction<DelegationToken>() {
@Override
public DelegationToken run() throws YarnRemoteException {
- GetDelegationTokenRequest request =
- Records.newRecord(GetDelegationTokenRequest.class);
- request.setRenewer("OneRenewerToRuleThemAll");
- return jobHistoryServer.getClientService().getClientHandler()
- .getDelegationToken(request).getDelegationToken();
+ GetDelegationTokenRequest request = Records
+ .newRecord(GetDelegationTokenRequest.class);
+ request.setRenewer(renewerString);
+ return hsService.getDelegationToken(request).getDelegationToken();
}
+
});
+ return token;
+ }
+
+ private long renewDelegationToken(final UserGroupInformation loggedInUser,
+ final MRClientProtocol hsService, final DelegationToken dToken)
+ throws IOException, InterruptedException {
+ long nextExpTime = loggedInUser.doAs(new PrivilegedExceptionAction<Long>() {
+
+ @Override
+ public Long run() throws YarnRemoteException {
+ RenewDelegationTokenRequest request = Records
+ .newRecord(RenewDelegationTokenRequest.class);
+ request.setDelegationToken(dToken);
+ return hsService.renewDelegationToken(request).getNextExpirationTime();
+ }
+ });
+ return nextExpTime;
+ }
+
+ private void cancelDelegationToken(final UserGroupInformation loggedInUser,
+ final MRClientProtocol hsService, final DelegationToken dToken)
+ throws IOException, InterruptedException {
+
+ loggedInUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws YarnRemoteException {
+ CancelDelegationTokenRequest request = Records
+ .newRecord(CancelDelegationTokenRequest.class);
+ request.setDelegationToken(dToken);
+ hsService.cancelDelegationToken(request);
+ return null;
+ }
+ });
+ }
+
+ private MRClientProtocol getMRClientProtocol(DelegationToken token,
+ final InetSocketAddress hsAddress, String user, final Configuration conf) {
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
+ ugi.addToken(ProtoUtils.convertFromProtoFormat(token, hsAddress));
- // Now try talking to JHS using the delegation token
- UserGroupInformation ugi =
- UserGroupInformation.createRemoteUser("TheDarkLord");
- ugi.addToken(ProtoUtils.convertFromProtoFormat(
- token, jobHistoryServer.getClientService().getBindAddress()));
final YarnRPC rpc = YarnRPC.create(conf);
- MRClientProtocol userUsingDT =
- ugi.doAs(new PrivilegedAction<MRClientProtocol>() {
+ MRClientProtocol hsWithDT = ugi
+ .doAs(new PrivilegedAction<MRClientProtocol>() {
+
@Override
public MRClientProtocol run() {
return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class,
- jobHistoryServer.getClientService().getBindAddress(), conf);
+ hsAddress, conf);
}
});
- GetJobReportRequest jobReportRequest =
- Records.newRecord(GetJobReportRequest.class);
- jobReportRequest.setJobId(MRBuilderUtils.newJobId(123456, 1, 1));
- try {
- userUsingDT.getJobReport(jobReportRequest);
- } catch (YarnRemoteException e) {
- Assert.assertEquals("Unknown job job_123456_0001", e.getMessage());
- }
+ return hsWithDT;
}
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml Tue Jan 22 19:33:02 2013
@@ -134,7 +134,15 @@
<effort>Max</effort>
</configuration>
</plugin>
-
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>src/main/java/org/apache/hadoop/examples/dancing/puzzle1.dta</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
</plugins>
</build>
</project>
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java Tue Jan 22 19:33:02 2013
@@ -21,6 +21,7 @@ package org.apache.hadoop.examples;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.RoundingMode;
+import java.util.Random;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
@@ -77,8 +78,7 @@ public class QuasiMonteCarlo extends Con
static final String DESCRIPTION
= "A map/reduce program that estimates Pi using a quasi-Monte Carlo method.";
/** tmp directory for input/output */
- static private final Path TMP_DIR = new Path(
- QuasiMonteCarlo.class.getSimpleName() + "_TMP_3_141592654");
+ static private final String TMP_DIR_PREFIX = QuasiMonteCarlo.class.getSimpleName();
/** 2-dimensional Halton sequence {H(i)},
* where H(i) is a 2-dimensional point and i >= 1 is the index.
@@ -228,9 +228,9 @@ public class QuasiMonteCarlo extends Con
@Override
public void cleanup(Context context) throws IOException {
//write output to a file
- Path outDir = new Path(TMP_DIR, "out");
- Path outFile = new Path(outDir, "reduce-out");
Configuration conf = context.getConfiguration();
+ Path outDir = new Path(conf.get(FileOutputFormat.OUTDIR));
+ Path outFile = new Path(outDir, "reduce-out");
FileSystem fileSys = FileSystem.get(conf);
SequenceFile.Writer writer = SequenceFile.createWriter(fileSys, conf,
outFile, LongWritable.class, LongWritable.class,
@@ -246,7 +246,7 @@ public class QuasiMonteCarlo extends Con
* @return the estimated value of Pi
*/
public static BigDecimal estimatePi(int numMaps, long numPoints,
- Configuration conf
+ Path tmpDir, Configuration conf
) throws IOException, ClassNotFoundException, InterruptedException {
Job job = new Job(conf);
//setup job conf
@@ -269,14 +269,14 @@ public class QuasiMonteCarlo extends Con
job.setSpeculativeExecution(false);
//setup input/output directories
- final Path inDir = new Path(TMP_DIR, "in");
- final Path outDir = new Path(TMP_DIR, "out");
+ final Path inDir = new Path(tmpDir, "in");
+ final Path outDir = new Path(tmpDir, "out");
FileInputFormat.setInputPaths(job, inDir);
FileOutputFormat.setOutputPath(job, outDir);
final FileSystem fs = FileSystem.get(conf);
- if (fs.exists(TMP_DIR)) {
- throw new IOException("Tmp directory " + fs.makeQualified(TMP_DIR)
+ if (fs.exists(tmpDir)) {
+ throw new IOException("Tmp directory " + fs.makeQualified(tmpDir)
+ " already exists. Please remove it first.");
}
if (!fs.mkdirs(inDir)) {
@@ -325,7 +325,7 @@ public class QuasiMonteCarlo extends Con
.multiply(BigDecimal.valueOf(numInside.get()))
.divide(numTotal, RoundingMode.HALF_UP);
} finally {
- fs.delete(TMP_DIR, true);
+ fs.delete(tmpDir, true);
}
}
@@ -344,12 +344,15 @@ public class QuasiMonteCarlo extends Con
final int nMaps = Integer.parseInt(args[0]);
final long nSamples = Long.parseLong(args[1]);
+ long now = System.currentTimeMillis();
+ int rand = new Random().nextInt(Integer.MAX_VALUE);
+ final Path tmpDir = new Path(TMP_DIR_PREFIX + "_" + now + "_" + rand);
System.out.println("Number of Maps = " + nMaps);
System.out.println("Samples per Map = " + nSamples);
System.out.println("Estimated value of Pi is "
- + estimatePi(nMaps, nSamples, getConf()));
+ + estimatePi(nMaps, nSamples, tmpDir, getConf()));
return 0;
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java Tue Jan 22 19:33:02 2013
@@ -174,16 +174,16 @@ public class DistributedPentomino extend
return 2;
}
// check for passed parameters, otherwise use defaults
- int width = PENT_WIDTH;
- int height = PENT_HEIGHT;
- int depth = PENT_DEPTH;
+ int width = conf.getInt(Pentomino.WIDTH, PENT_WIDTH);
+ int height = conf.getInt(Pentomino.HEIGHT, PENT_HEIGHT);
+ int depth = conf.getInt(Pentomino.DEPTH, PENT_DEPTH);
for (int i = 0; i < args.length; i++) {
if (args[i].equalsIgnoreCase("-depth")) {
- depth = Integer.parseInt(args[i++].trim());
+ depth = Integer.parseInt(args[++i].trim());
} else if (args[i].equalsIgnoreCase("-height")) {
- height = Integer.parseInt(args[i++].trim());
+ height = Integer.parseInt(args[++i].trim());
} else if (args[i].equalsIgnoreCase("-width") ) {
- width = Integer.parseInt(args[i++].trim());
+ width = Integer.parseInt(args[++i].trim());
}
}
// now set the values within conf for M/R tasks to read, this
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/TestWordStats.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/TestWordStats.java?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/TestWordStats.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/TestWordStats.java Tue Jan 22 19:33:02 2013
@@ -1,3 +1,20 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
package org.apache.hadoop.examples;
import static org.junit.Assert.assertEquals;
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/pom.xml?rev=1437113&r1=1437112&r2=1437113&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/pom.xml (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/pom.xml Tue Jan 22 19:33:02 2013
@@ -214,9 +214,11 @@ with tarfile.open(tar_name, "w:gz") as t
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
- <includes>
- <include>pom.xml</include>
- </includes>
+ <excludes>
+ <exclude>.eclipse.templates/</exclude>
+ <exclude>CHANGES.txt</exclude>
+ <exclude>lib/jdiff/**</exclude>
+ </excludes>
</configuration>
</plugin>
<plugin>