You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2016/05/03 18:59:42 UTC
[38/50] hadoop git commit: HADOOP-12563. Updated utility (dtutil) to
create/modify token files. Contributed by Matthew Paduano
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2753185a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tools/TestCommandShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tools/TestCommandShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tools/TestCommandShell.java
new file mode 100644
index 0000000..6067918
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/tools/TestCommandShell.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tools;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.tools.CommandShell;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestCommandShell {
+
+ public class Example extends CommandShell {
+ public static final String EXAMPLE = "example";
+ public static final String HELLO = "hello";
+ public static final String HELLO_MSG = "hello is running";
+ public static final String GOODBYE = "goodbye";
+ public static final String GOODBYE_MSG = "goodbye is running";
+
+ public String[] savedArgs = null;
+
+ @Override
+ protected int init(String[] args) throws Exception {
+ String command = args[0];
+ if (command.equals(HELLO)) {
+ setSubCommand(new Hello());
+ } else if (command.equals(GOODBYE)) {
+ setSubCommand(new Goodbye());
+ } else{
+ return 1;
+ }
+ savedArgs = args;
+ return 0;
+ }
+
+ public String getCommandUsage() {
+ return EXAMPLE;
+ }
+
+ public class Hello extends SubCommand {
+ public static final String HELLO_USAGE = EXAMPLE + " hello";
+ @Override
+ public boolean validate() {
+ return savedArgs.length == 1;
+ }
+ @Override
+ public void execute() throws Exception {
+ System.out.println(HELLO_MSG);
+ }
+ @Override
+ public String getUsage() {
+ return HELLO_USAGE;
+ }
+ }
+
+ public class Goodbye extends SubCommand {
+ public static final String GOODBYE_USAGE = EXAMPLE + " goodbye";
+ @Override
+ public void execute() throws Exception {
+ System.out.println(GOODBYE_MSG);
+ }
+ @Override
+ public String getUsage() {
+ return GOODBYE_USAGE;
+ }
+ }
+ }
+
+ private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+
+ private String outMsg(String message) {
+ return "OUT:\n" + outContent.toString() + "\n" + message;
+ }
+
+ @Before
+ public void setup() throws Exception {
+ System.setOut(new PrintStream(outContent));
+ }
+
+ @Test
+ public void testCommandShellExample() throws Exception {
+ Example ex = new Example();
+ ex.setConf(new Configuration());
+ int rc = 0;
+
+ outContent.reset();
+ String[] args1 = {"hello"};
+ rc = ex.run(args1);
+ assertEquals(outMsg("test exit code - normal hello"), 0, rc);
+ assertTrue(outMsg("test normal hello message"),
+ outContent.toString().contains(Example.HELLO_MSG));
+
+ outContent.reset();
+ String[] args2 = {"hello", "x"};
+ rc = ex.run(args2);
+ assertEquals(outMsg("test exit code - bad hello"), 1, rc);
+ assertTrue(outMsg("test bad hello message"),
+ outContent.toString().contains(Example.Hello.HELLO_USAGE));
+
+ outContent.reset();
+ String[] args3 = {"goodbye"};
+ rc = ex.run(args3);
+ assertEquals(outMsg("test exit code - normal goodbye"), 0, rc);
+ assertTrue(outMsg("test normal goodbye message"),
+ outContent.toString().contains(Example.GOODBYE_MSG));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2753185a/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher b/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
new file mode 100644
index 0000000..0f4a310
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
@@ -0,0 +1,14 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+org.apache.hadoop.security.token.TestDtFetcher
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2753185a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsDtFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsDtFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsDtFetcher.java
new file mode 100644
index 0000000..02aa4b9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsDtFetcher.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.DtFetcher;
+import org.apache.hadoop.security.token.Token;
+
+
+/**
+ * DtFetcher is an interface which permits the abstraction and separation of
+ * delegation token fetch implementaions across different packages and
+ * compilation units. Resolution of fetcher impl will be done at runtime.
+ */
+public class HdfsDtFetcher implements DtFetcher {
+ private static final Log LOG = LogFactory.getLog(HdfsDtFetcher.class);
+
+ private static final String SERVICE_NAME = HdfsConstants.HDFS_URI_SCHEME;
+
+ private static final String FETCH_FAILED = "Fetch of delegation token failed";
+
+ /**
+ * Returns the service name for HDFS, which is also a valid URL prefix.
+ */
+ public Text getServiceName() {
+ return new Text(SERVICE_NAME);
+ }
+
+ public boolean isTokenRequired() {
+ return UserGroupInformation.isSecurityEnabled();
+ }
+
+ /**
+ * Returns Token object via FileSystem, null if bad argument.
+ * @param conf - a Configuration object used with FileSystem.get()
+ * @param creds - a Credentials object to which token(s) will be added
+ * @param renewer - the renewer to send with the token request
+ * @param url - the URL to which the request is sent
+ * @return a Token, or null if fetch fails.
+ */
+ public Token<?> addDelegationTokens(Configuration conf, Credentials creds,
+ String renewer, String url) throws Exception {
+ if (!url.startsWith(getServiceName().toString())) {
+ url = getServiceName().toString() + "://" + url;
+ }
+ FileSystem fs = FileSystem.get(URI.create(url), conf);
+ Token<?> token = fs.getDelegationToken(renewer);
+ if (token == null) {
+ LOG.error(FETCH_FAILED);
+ throw new IOException(FETCH_FAILED);
+ }
+ creds.addToken(token.getService(), token);
+ return token;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2753185a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/SWebHdfsDtFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/SWebHdfsDtFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/SWebHdfsDtFetcher.java
new file mode 100644
index 0000000..46f9b00
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/SWebHdfsDtFetcher.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.hdfs.web.WebHdfsConstants;
+import org.apache.hadoop.io.Text;
+
+/**
+ * DtFetcher for SWebHdfsFileSystem using the base class HdfsDtFetcher impl.
+ */
+public class SWebHdfsDtFetcher extends HdfsDtFetcher {
+ private static final Log LOG = LogFactory.getLog(SWebHdfsDtFetcher.class);
+
+ private static final String SERVICE_NAME = WebHdfsConstants.SWEBHDFS_SCHEME;
+
+ @Override
+ public Text getServiceName() {
+ return new Text(SERVICE_NAME);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2753185a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/WebHdfsDtFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/WebHdfsDtFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/WebHdfsDtFetcher.java
new file mode 100644
index 0000000..c2bb852
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/WebHdfsDtFetcher.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.hdfs.web.WebHdfsConstants;
+import org.apache.hadoop.io.Text;
+
+/**
+ * DtFetcher for WebHdfsFileSystem using the base class HdfsDtFetcher impl.
+ */
+public class WebHdfsDtFetcher extends HdfsDtFetcher {
+ private static final Log LOG = LogFactory.getLog(WebHdfsDtFetcher.class);
+
+ private static final String SERVICE_NAME = WebHdfsConstants.WEBHDFS_SCHEME;
+
+ @Override
+ public Text getServiceName() {
+ return new Text(SERVICE_NAME);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2753185a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
index 39821aa..f24b50e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
@@ -179,7 +179,8 @@ public class DelegationTokenFetcher {
if (null != token) {
Credentials cred = new Credentials();
cred.addToken(token.getService(), token);
- cred.writeTokenStorageFile(tokenFile, conf);
+ // dtutil is replacing this tool; preserve legacy functionality
+ cred.writeLegacyTokenStorageFile(tokenFile, conf);
if (LOG.isDebugEnabled()) {
LOG.debug("Fetched token " + fs.getUri() + " for " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2753185a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
new file mode 100644
index 0000000..a4217fb
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.hadoop.hdfs.HdfsDtFetcher
+org.apache.hadoop.hdfs.WebHdfsDtFetcher
+org.apache.hadoop.hdfs.SWebHdfsDtFetcher
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org