You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ni...@apache.org on 2008/03/08 20:20:12 UTC
svn commit: r635051 - in /hadoop/core/trunk/src:
java/org/apache/hadoop/io/MultipleIOException.java
test/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java
Author: nigel
Date: Sat Mar 8 11:20:09 2008
New Revision: 635051
URL: http://svn.apache.org/viewvc?rev=635051&view=rev
Log:
HADOOP-2915. Fixed FileSystem.CACHE so that a username is included in the cache key. Contributed by Tsz Wo (Nicholas) SZE.
Added:
hadoop/core/trunk/src/java/org/apache/hadoop/io/MultipleIOException.java (with props)
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java (with props)
Added: hadoop/core/trunk/src/java/org/apache/hadoop/io/MultipleIOException.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/io/MultipleIOException.java?rev=635051&view=auto
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/io/MultipleIOException.java (added)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/io/MultipleIOException.java Sat Mar 8 11:20:09 2008
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io;
+
+import java.io.IOException;
+import java.util.List;
+
+/** Encapsulate a list of {@link IOException} into an {@link IOException} */
+public class MultipleIOException extends IOException {
+ /** Require by {@link java.io.Serializable} */
+ private static final long serialVersionUID = 1L;
+
+ private final List<IOException> exceptions;
+
+ /** Constructor is private, use {@link #createIOException(List)}. */
+ private MultipleIOException(List<IOException> exceptions) {
+ super(exceptions.size() + " exceptions " + exceptions);
+ this.exceptions = exceptions;
+ }
+
+ /** @return the underlying exceptions */
+ public List<IOException> getExceptions() {return exceptions;}
+
+ /** A convenient method to create an {@link IOException}. */
+ public static IOException createIOException(List<IOException> exceptions) {
+ if (exceptions == null || exceptions.isEmpty()) {
+ return null;
+ }
+ if (exceptions.size() == 1) {
+ return exceptions.get(0);
+ }
+ return new MultipleIOException(exceptions);
+ }
+}
\ No newline at end of file
Propchange: hadoop/core/trunk/src/java/org/apache/hadoop/io/MultipleIOException.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: hadoop/core/trunk/src/java/org/apache/hadoop/io/MultipleIOException.java
------------------------------------------------------------------------------
svn:keywords = Id Revision HeadURL
Added: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java?rev=635051&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java (added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java Sat Mar 8 11:20:09 2008
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import java.io.*;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.dfs.MiniDFSCluster;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.*;
+
+/**
+ * A JUnit test to test Mini Map-Reduce Cluster with Mini-DFS.
+ */
+public class TestMiniMRWithDFSWithDistinctUsers extends TestCase {
+ static final long now = System.currentTimeMillis();
+ static final UnixUserGroupInformation DFS_UGI = createUGI("dfs", true);
+ static final UnixUserGroupInformation PI_UGI = createUGI("pi", false);
+ static final UnixUserGroupInformation WC_UGI = createUGI("wc", false);
+
+ static UnixUserGroupInformation createUGI(String name, boolean issuper) {
+ String username = name + now;
+ String group = issuper? "supergroup": username;
+ return UnixUserGroupInformation.createImmutable(
+ new String[]{username, group});
+ }
+
+ static JobConf createJobConf(MiniMRCluster mr, UnixUserGroupInformation ugi) {
+ JobConf jobconf = mr.createJobConf();
+ UnixUserGroupInformation.saveToConf(jobconf,
+ UnixUserGroupInformation.UGI_PROPERTY_NAME, ugi);
+ return jobconf;
+ }
+
+ static void mkdir(FileSystem fs, String dir) throws IOException {
+ Path p = new Path(dir);
+ fs.mkdirs(p);
+ fs.setPermission(p, new FsPermission((short)0777));
+ }
+
+ public void testDistinctUsers() throws Exception {
+ MiniDFSCluster dfs = null;
+ MiniMRCluster mr = null;
+ try {
+ Configuration conf = new Configuration();
+ UnixUserGroupInformation.saveToConf(conf,
+ UnixUserGroupInformation.UGI_PROPERTY_NAME, DFS_UGI);
+ dfs = new MiniDFSCluster(conf, 4, true, null);
+ FileSystem fs = dfs.getFileSystem();
+ mkdir(fs, "/user");
+ mkdir(fs, "/mapred");
+
+ UnixUserGroupInformation MR_UGI = createUGI(
+ UnixUserGroupInformation.login().getUserName(), true);
+ mr = new MiniMRCluster(0, 0, 4, dfs.getFileSystem().getUri().toString(),
+ false, 1, null, null, MR_UGI);
+
+ JobConf pi = createJobConf(mr, PI_UGI);
+ TestMiniMRWithDFS.runPI(mr, pi);
+
+ JobConf wc = createJobConf(mr, WC_UGI);
+ TestMiniMRWithDFS.runWordCount(mr, wc);
+ } finally {
+ if (dfs != null) { dfs.shutdown(); }
+ if (mr != null) { mr.shutdown();}
+ }
+ }
+}
\ No newline at end of file
Propchange: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java
------------------------------------------------------------------------------
svn:keywords = Id Revision HeadURL