You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/02/27 00:17:19 UTC
[01/17] hadoop git commit: HADOOP-11629. WASB filesystem should not
start BandwidthGaugeUpdater if fs.azure.skip.metrics set to true. Contributed
by Shanyu Zhao.
Repository: hadoop
Updated Branches:
refs/heads/YARN-2928 6bb198e72 -> 7788f9773
HADOOP-11629. WASB filesystem should not start BandwidthGaugeUpdater if fs.azure.skip.metrics set to true. Contributed by Shanyu Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5731c0e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5731c0e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5731c0e0
Branch: refs/heads/YARN-2928
Commit: 5731c0e0d08c3048fafdf62a14ca7611be4df5d7
Parents: 1a68fc4
Author: cnauroth <cn...@apache.org>
Authored: Wed Feb 25 09:08:55 2015 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Wed Feb 25 09:08:55 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../fs/azure/AzureNativeFileSystemStore.java | 29 ++++++++++++++------
2 files changed, 23 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5731c0e0/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 988eed0..0d452f7 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1002,6 +1002,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11480. Typo in hadoop-aws/index.md uses wrong scheme for
test.fs.s3.name. (Ted Yu via aajisaka)
+ HADOOP-11629. WASB filesystem should not start BandwidthGaugeUpdater if
+ fs.azure.skip.metrics set to true. (Shanyu Zhao via cnauroth)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5731c0e0/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index 2412698..6bed8bb 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -387,9 +387,8 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
if (null == instrumentation) {
throw new IllegalArgumentException("Null instrumentation");
}
-
this.instrumentation = instrumentation;
- this.bandwidthGaugeUpdater = new BandwidthGaugeUpdater(instrumentation);
+
if (null == this.storageInteractionLayer) {
this.storageInteractionLayer = new StorageInterfaceImpl();
}
@@ -405,7 +404,13 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
//
if (null == conf) {
throw new IllegalArgumentException(
- "Cannot initialize WASB file system, URI is null");
+ "Cannot initialize WASB file system, conf is null");
+ }
+
+ if(!conf.getBoolean(
+ NativeAzureFileSystem.SKIP_AZURE_METRICS_PROPERTY_NAME, false)) {
+ //If not skip azure metrics, create bandwidthGaugeUpdater
+ this.bandwidthGaugeUpdater = new BandwidthGaugeUpdater(instrumentation);
}
// Incoming parameters validated. Capture the URI and the job configuration
@@ -1782,11 +1787,14 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
selfThrottlingWriteFactor);
}
- ResponseReceivedMetricUpdater.hook(
- operationContext,
- instrumentation,
- bandwidthGaugeUpdater);
-
+ if(bandwidthGaugeUpdater != null) {
+ //bandwidthGaugeUpdater is null when we config to skip azure metrics
+ ResponseReceivedMetricUpdater.hook(
+ operationContext,
+ instrumentation,
+ bandwidthGaugeUpdater);
+ }
+
// Bind operation context to receive send request callbacks on this operation.
// If reads concurrent to OOB writes are allowed, the interception will reset
// the conditional header on all Azure blob storage read requests.
@@ -2561,7 +2569,10 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
@Override
public void close() {
- bandwidthGaugeUpdater.close();
+ if(bandwidthGaugeUpdater != null) {
+ bandwidthGaugeUpdater.close();
+ bandwidthGaugeUpdater = null;
+ }
}
// Finalizer to ensure complete shutdown
[07/17] hadoop git commit: HADOOP-11620. Add support for load
balancing across a group of KMS for HA. Contributed by Arun Suresh.
Posted by zj...@apache.org.
HADOOP-11620. Add support for load balancing across a group of KMS for HA. Contributed by Arun Suresh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71385f9b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71385f9b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71385f9b
Branch: refs/heads/YARN-2928
Commit: 71385f9b70e22618db3f3d2b2c6dca3b1e82c317
Parents: 725cc49
Author: Andrew Wang <wa...@apache.org>
Authored: Wed Feb 25 21:15:44 2015 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Wed Feb 25 21:16:37 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../crypto/key/kms/KMSClientProvider.java | 84 ++++-
.../key/kms/LoadBalancingKMSClientProvider.java | 347 +++++++++++++++++++
.../kms/TestLoadBalancingKMSClientProvider.java | 166 +++++++++
.../hadoop/crypto/key/kms/server/TestKMS.java | 114 +++---
5 files changed, 654 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71385f9b/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0d452f7..39062a8 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -648,6 +648,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11506. Configuration variable expansion regex expensive for long
values. (Gera Shegalov via gera)
+ HADOOP-11620. Add support for load balancing across a group of KMS for HA.
+ (Arun Suresh via wang)
+
BUG FIXES
HADOOP-11512. Use getTrimmedStrings when reading serialization keys
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71385f9b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 97ab253..223e69a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -52,6 +52,7 @@ import java.io.Writer;
import java.lang.reflect.UndeclaredThrowableException;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
+import java.net.MalformedURLException;
import java.net.SocketTimeoutException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -74,6 +75,7 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
/**
* KMS client <code>KeyProvider</code> implementation.
@@ -221,14 +223,71 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
*/
public static class Factory extends KeyProviderFactory {
+ /**
+ * This provider expects URIs in the following form :
+ * kms://<PROTO>@<AUTHORITY>/<PATH>
+ *
+ * where :
+ * - PROTO = http or https
+ * - AUTHORITY = <HOSTS>[:<PORT>]
+ * - HOSTS = <HOSTNAME>[;<HOSTS>]
+ * - HOSTNAME = string
+ * - PORT = integer
+ *
+ * If multiple hosts are provider, the Factory will create a
+ * {@link LoadBalancingKMSClientProvider} that round-robins requests
+ * across the provided list of hosts.
+ */
@Override
- public KeyProvider createProvider(URI providerName, Configuration conf)
+ public KeyProvider createProvider(URI providerUri, Configuration conf)
throws IOException {
- if (SCHEME_NAME.equals(providerName.getScheme())) {
- return new KMSClientProvider(providerName, conf);
+ if (SCHEME_NAME.equals(providerUri.getScheme())) {
+ URL origUrl = new URL(extractKMSPath(providerUri).toString());
+ String authority = origUrl.getAuthority();
+ // check for ';' which delimits the backup hosts
+ if (Strings.isNullOrEmpty(authority)) {
+ throw new IOException(
+ "No valid authority in kms uri [" + origUrl + "]");
+ }
+ // Check if port is present in authority
+ // In the current scheme, all hosts have to run on the same port
+ int port = -1;
+ String hostsPart = authority;
+ if (authority.contains(":")) {
+ String[] t = authority.split(":");
+ try {
+ port = Integer.parseInt(t[1]);
+ } catch (Exception e) {
+ throw new IOException(
+ "Could not parse port in kms uri [" + origUrl + "]");
+ }
+ hostsPart = t[0];
+ }
+ return createProvider(providerUri, conf, origUrl, port, hostsPart);
}
return null;
}
+
+ private KeyProvider createProvider(URI providerUri, Configuration conf,
+ URL origUrl, int port, String hostsPart) throws IOException {
+ String[] hosts = hostsPart.split(";");
+ if (hosts.length == 1) {
+ return new KMSClientProvider(providerUri, conf);
+ } else {
+ KMSClientProvider[] providers = new KMSClientProvider[hosts.length];
+ for (int i = 0; i < hosts.length; i++) {
+ try {
+ providers[i] =
+ new KMSClientProvider(
+ new URI("kms", origUrl.getProtocol(), hosts[i], port,
+ origUrl.getPath(), null, null), conf);
+ } catch (URISyntaxException e) {
+ throw new IOException("Could not instantiate KMSProvider..", e);
+ }
+ }
+ return new LoadBalancingKMSClientProvider(providers, conf);
+ }
+ }
}
public static <T> T checkNotNull(T o, String name)
@@ -302,10 +361,8 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
public KMSClientProvider(URI uri, Configuration conf) throws IOException {
super(conf);
- Path path = ProviderUtils.unnestUri(uri);
- URL url = path.toUri().toURL();
- kmsUrl = createServiceURL(url);
- if ("https".equalsIgnoreCase(url.getProtocol())) {
+ kmsUrl = createServiceURL(extractKMSPath(uri));
+ if ("https".equalsIgnoreCase(new URL(kmsUrl).getProtocol())) {
sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
try {
sslFactory.init();
@@ -346,8 +403,12 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
.getCurrentUser();
}
- private String createServiceURL(URL url) throws IOException {
- String str = url.toExternalForm();
+ private static Path extractKMSPath(URI uri) throws MalformedURLException, IOException {
+ return ProviderUtils.unnestUri(uri);
+ }
+
+ private static String createServiceURL(Path path) throws IOException {
+ String str = new URL(path.toString()).toExternalForm();
if (str.endsWith("/")) {
str = str.substring(0, str.length() - 1);
}
@@ -853,4 +914,9 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
}
}
}
+
+ @VisibleForTesting
+ String getKMSUrl() {
+ return kmsUrl;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71385f9b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
new file mode 100644
index 0000000..c1579e7
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
@@ -0,0 +1,347 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.crypto.key.kms;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
+import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * A simple LoadBalancing KMSClientProvider that round-robins requests
+ * across a provided array of KMSClientProviders. It also retries failed
+ * requests on the next available provider in the load balancer group. It
+ * only retries failed requests that result in an IOException, sending back
+ * all other Exceptions to the caller without retry.
+ */
+public class LoadBalancingKMSClientProvider extends KeyProvider implements
+ CryptoExtension,
+ KeyProviderDelegationTokenExtension.DelegationTokenExtension {
+
+ public static Logger LOG =
+ LoggerFactory.getLogger(LoadBalancingKMSClientProvider.class);
+
+ static interface ProviderCallable<T> {
+ public T call(KMSClientProvider provider) throws IOException, Exception;
+ }
+
+ @SuppressWarnings("serial")
+ static class WrapperException extends RuntimeException {
+ public WrapperException(Throwable cause) {
+ super(cause);
+ }
+ }
+
+ private final KMSClientProvider[] providers;
+ private final AtomicInteger currentIdx;
+
+ public LoadBalancingKMSClientProvider(KMSClientProvider[] providers,
+ Configuration conf) {
+ this(shuffle(providers), Time.monotonicNow(), conf);
+ }
+
+ @VisibleForTesting
+ LoadBalancingKMSClientProvider(KMSClientProvider[] providers, long seed,
+ Configuration conf) {
+ super(conf);
+ this.providers = providers;
+ this.currentIdx = new AtomicInteger((int)(seed % providers.length));
+ }
+
+ @VisibleForTesting
+ KMSClientProvider[] getProviders() {
+ return providers;
+ }
+
+ private <T> T doOp(ProviderCallable<T> op, int currPos)
+ throws IOException {
+ IOException ex = null;
+ for (int i = 0; i < providers.length; i++) {
+ KMSClientProvider provider = providers[(currPos + i) % providers.length];
+ try {
+ return op.call(provider);
+ } catch (IOException ioe) {
+ LOG.warn("KMS provider at [{}] threw an IOException [{}]!!",
+ provider.getKMSUrl(), ioe.getMessage());
+ ex = ioe;
+ } catch (Exception e) {
+ if (e instanceof RuntimeException) {
+ throw (RuntimeException)e;
+ } else {
+ throw new WrapperException(e);
+ }
+ }
+ }
+ if (ex != null) {
+ LOG.warn("Aborting since the Request has failed with all KMS"
+ + " providers in the group. !!");
+ throw ex;
+ }
+ throw new IOException("No providers configured !!");
+ }
+
+ private int nextIdx() {
+ while (true) {
+ int current = currentIdx.get();
+ int next = (current + 1) % providers.length;
+ if (currentIdx.compareAndSet(current, next)) {
+ return current;
+ }
+ }
+ }
+
+ @Override
+ public Token<?>[]
+ addDelegationTokens(final String renewer, final Credentials credentials)
+ throws IOException {
+ return doOp(new ProviderCallable<Token<?>[]>() {
+ @Override
+ public Token<?>[] call(KMSClientProvider provider) throws IOException {
+ return provider.addDelegationTokens(renewer, credentials);
+ }
+ }, nextIdx());
+ }
+
+ // This request is sent to all providers in the load-balancing group
+ @Override
+ public void warmUpEncryptedKeys(String... keyNames) throws IOException {
+ for (KMSClientProvider provider : providers) {
+ try {
+ provider.warmUpEncryptedKeys(keyNames);
+ } catch (IOException ioe) {
+ LOG.error(
+ "Error warming up keys for provider with url"
+ + "[" + provider.getKMSUrl() + "]");
+ }
+ }
+ }
+
+ // This request is sent to all providers in the load-balancing group
+ @Override
+ public void drain(String keyName) {
+ for (KMSClientProvider provider : providers) {
+ provider.drain(keyName);
+ }
+ }
+
+ @Override
+ public EncryptedKeyVersion
+ generateEncryptedKey(final String encryptionKeyName)
+ throws IOException, GeneralSecurityException {
+ try {
+ return doOp(new ProviderCallable<EncryptedKeyVersion>() {
+ @Override
+ public EncryptedKeyVersion call(KMSClientProvider provider)
+ throws IOException, GeneralSecurityException {
+ return provider.generateEncryptedKey(encryptionKeyName);
+ }
+ }, nextIdx());
+ } catch (WrapperException we) {
+ throw (GeneralSecurityException) we.getCause();
+ }
+ }
+
+ @Override
+ public KeyVersion
+ decryptEncryptedKey(final EncryptedKeyVersion encryptedKeyVersion)
+ throws IOException, GeneralSecurityException {
+ try {
+ return doOp(new ProviderCallable<KeyVersion>() {
+ @Override
+ public KeyVersion call(KMSClientProvider provider)
+ throws IOException, GeneralSecurityException {
+ return provider.decryptEncryptedKey(encryptedKeyVersion);
+ }
+ }, nextIdx());
+ } catch (WrapperException we) {
+ throw (GeneralSecurityException)we.getCause();
+ }
+ }
+
+ @Override
+ public KeyVersion getKeyVersion(final String versionName) throws IOException {
+ return doOp(new ProviderCallable<KeyVersion>() {
+ @Override
+ public KeyVersion call(KMSClientProvider provider) throws IOException {
+ return provider.getKeyVersion(versionName);
+ }
+ }, nextIdx());
+ }
+
+ @Override
+ public List<String> getKeys() throws IOException {
+ return doOp(new ProviderCallable<List<String>>() {
+ @Override
+ public List<String> call(KMSClientProvider provider) throws IOException {
+ return provider.getKeys();
+ }
+ }, nextIdx());
+ }
+
+ @Override
+ public Metadata[] getKeysMetadata(final String... names) throws IOException {
+ return doOp(new ProviderCallable<Metadata[]>() {
+ @Override
+ public Metadata[] call(KMSClientProvider provider) throws IOException {
+ return provider.getKeysMetadata(names);
+ }
+ }, nextIdx());
+ }
+
+ @Override
+ public List<KeyVersion> getKeyVersions(final String name) throws IOException {
+ return doOp(new ProviderCallable<List<KeyVersion>>() {
+ @Override
+ public List<KeyVersion> call(KMSClientProvider provider)
+ throws IOException {
+ return provider.getKeyVersions(name);
+ }
+ }, nextIdx());
+ }
+
+ @Override
+ public KeyVersion getCurrentKey(final String name) throws IOException {
+ return doOp(new ProviderCallable<KeyVersion>() {
+ @Override
+ public KeyVersion call(KMSClientProvider provider) throws IOException {
+ return provider.getCurrentKey(name);
+ }
+ }, nextIdx());
+ }
+ @Override
+ public Metadata getMetadata(final String name) throws IOException {
+ return doOp(new ProviderCallable<Metadata>() {
+ @Override
+ public Metadata call(KMSClientProvider provider) throws IOException {
+ return provider.getMetadata(name);
+ }
+ }, nextIdx());
+ }
+
+ @Override
+ public KeyVersion createKey(final String name, final byte[] material,
+ final Options options) throws IOException {
+ return doOp(new ProviderCallable<KeyVersion>() {
+ @Override
+ public KeyVersion call(KMSClientProvider provider) throws IOException {
+ return provider.createKey(name, material, options);
+ }
+ }, nextIdx());
+ }
+
+ @Override
+ public KeyVersion createKey(final String name, final Options options)
+ throws NoSuchAlgorithmException, IOException {
+ try {
+ return doOp(new ProviderCallable<KeyVersion>() {
+ @Override
+ public KeyVersion call(KMSClientProvider provider) throws IOException,
+ NoSuchAlgorithmException {
+ return provider.createKey(name, options);
+ }
+ }, nextIdx());
+ } catch (WrapperException e) {
+ throw (NoSuchAlgorithmException)e.getCause();
+ }
+ }
+ @Override
+ public void deleteKey(final String name) throws IOException {
+ doOp(new ProviderCallable<Void>() {
+ @Override
+ public Void call(KMSClientProvider provider) throws IOException {
+ provider.deleteKey(name);
+ return null;
+ }
+ }, nextIdx());
+ }
+ @Override
+ public KeyVersion rollNewVersion(final String name, final byte[] material)
+ throws IOException {
+ return doOp(new ProviderCallable<KeyVersion>() {
+ @Override
+ public KeyVersion call(KMSClientProvider provider) throws IOException {
+ return provider.rollNewVersion(name, material);
+ }
+ }, nextIdx());
+ }
+
+ @Override
+ public KeyVersion rollNewVersion(final String name)
+ throws NoSuchAlgorithmException, IOException {
+ try {
+ return doOp(new ProviderCallable<KeyVersion>() {
+ @Override
+ public KeyVersion call(KMSClientProvider provider) throws IOException,
+ NoSuchAlgorithmException {
+ return provider.rollNewVersion(name);
+ }
+ }, nextIdx());
+ } catch (WrapperException e) {
+ throw (NoSuchAlgorithmException)e.getCause();
+ }
+ }
+
+ // Close all providers in the LB group
+ @Override
+ public void close() throws IOException {
+ for (KMSClientProvider provider : providers) {
+ try {
+ provider.close();
+ } catch (IOException ioe) {
+ LOG.error("Error closing provider with url"
+ + "[" + provider.getKMSUrl() + "]");
+ }
+ }
+ }
+
+
+ @Override
+ public void flush() throws IOException {
+ for (KMSClientProvider provider : providers) {
+ try {
+ provider.flush();
+ } catch (IOException ioe) {
+ LOG.error("Error flushing provider with url"
+ + "[" + provider.getKMSUrl() + "]");
+ }
+ }
+ }
+
+ private static KMSClientProvider[] shuffle(KMSClientProvider[] providers) {
+ List<KMSClientProvider> list = Arrays.asList(providers);
+ Collections.shuffle(list);
+ return list.toArray(providers);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71385f9b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
new file mode 100644
index 0000000..08a3d93
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
@@ -0,0 +1,166 @@
+/** when(p1.getKMSUrl()).thenReturn("p1");
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.net.URI;
+import java.security.NoSuchAlgorithmException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProvider.Options;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import com.google.common.collect.Sets;
+
+public class TestLoadBalancingKMSClientProvider {
+
+ @Test
+ public void testCreation() throws Exception {
+ Configuration conf = new Configuration();
+ KeyProvider kp = new KMSClientProvider.Factory().createProvider(new URI(
+ "kms://http@host1/kms/foo"), conf);
+ assertTrue(kp instanceof KMSClientProvider);
+ assertEquals("http://host1/kms/foo/v1/",
+ ((KMSClientProvider) kp).getKMSUrl());
+
+ kp = new KMSClientProvider.Factory().createProvider(new URI(
+ "kms://http@host1;host2;host3/kms/foo"), conf);
+ assertTrue(kp instanceof LoadBalancingKMSClientProvider);
+ KMSClientProvider[] providers =
+ ((LoadBalancingKMSClientProvider) kp).getProviders();
+ assertEquals(3, providers.length);
+ assertEquals(Sets.newHashSet("http://host1/kms/foo/v1/",
+ "http://host2/kms/foo/v1/",
+ "http://host3/kms/foo/v1/"),
+ Sets.newHashSet(providers[0].getKMSUrl(),
+ providers[1].getKMSUrl(),
+ providers[2].getKMSUrl()));
+
+ kp = new KMSClientProvider.Factory().createProvider(new URI(
+ "kms://http@host1;host2;host3:16000/kms/foo"), conf);
+ assertTrue(kp instanceof LoadBalancingKMSClientProvider);
+ providers =
+ ((LoadBalancingKMSClientProvider) kp).getProviders();
+ assertEquals(3, providers.length);
+ assertEquals(Sets.newHashSet("http://host1:16000/kms/foo/v1/",
+ "http://host2:16000/kms/foo/v1/",
+ "http://host3:16000/kms/foo/v1/"),
+ Sets.newHashSet(providers[0].getKMSUrl(),
+ providers[1].getKMSUrl(),
+ providers[2].getKMSUrl()));
+ }
+
+ @Test
+ public void testLoadBalancing() throws Exception {
+ Configuration conf = new Configuration();
+ KMSClientProvider p1 = mock(KMSClientProvider.class);
+ when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenReturn(
+ new KMSClientProvider.KMSKeyVersion("p1", "v1", new byte[0]));
+ KMSClientProvider p2 = mock(KMSClientProvider.class);
+ when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenReturn(
+ new KMSClientProvider.KMSKeyVersion("p2", "v2", new byte[0]));
+ KMSClientProvider p3 = mock(KMSClientProvider.class);
+ when(p3.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenReturn(
+ new KMSClientProvider.KMSKeyVersion("p3", "v3", new byte[0]));
+ KeyProvider kp = new LoadBalancingKMSClientProvider(
+ new KMSClientProvider[] { p1, p2, p3 }, 0, conf);
+ assertEquals("p1", kp.createKey("test1", new Options(conf)).getName());
+ assertEquals("p2", kp.createKey("test2", new Options(conf)).getName());
+ assertEquals("p3", kp.createKey("test3", new Options(conf)).getName());
+ assertEquals("p1", kp.createKey("test4", new Options(conf)).getName());
+ }
+
+ @Test
+ public void testLoadBalancingWithFailure() throws Exception {
+ Configuration conf = new Configuration();
+ KMSClientProvider p1 = mock(KMSClientProvider.class);
+ when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenReturn(
+ new KMSClientProvider.KMSKeyVersion("p1", "v1", new byte[0]));
+ when(p1.getKMSUrl()).thenReturn("p1");
+ // This should not be retried
+ KMSClientProvider p2 = mock(KMSClientProvider.class);
+ when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new NoSuchAlgorithmException("p2"));
+ when(p2.getKMSUrl()).thenReturn("p2");
+ KMSClientProvider p3 = mock(KMSClientProvider.class);
+ when(p3.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenReturn(
+ new KMSClientProvider.KMSKeyVersion("p3", "v3", new byte[0]));
+ when(p3.getKMSUrl()).thenReturn("p3");
+ // This should be retried
+ KMSClientProvider p4 = mock(KMSClientProvider.class);
+ when(p4.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new IOException("p4"));
+ when(p4.getKMSUrl()).thenReturn("p4");
+ KeyProvider kp = new LoadBalancingKMSClientProvider(
+ new KMSClientProvider[] { p1, p2, p3, p4 }, 0, conf);
+
+ assertEquals("p1", kp.createKey("test4", new Options(conf)).getName());
+ // Exceptions other than IOExceptions will not be retried
+ try {
+ kp.createKey("test1", new Options(conf)).getName();
+ fail("Should fail since its not an IOException");
+ } catch (Exception e) {
+ assertTrue(e instanceof NoSuchAlgorithmException);
+ }
+ assertEquals("p3", kp.createKey("test2", new Options(conf)).getName());
+ // IOException will trigger retry in next provider
+ assertEquals("p1", kp.createKey("test3", new Options(conf)).getName());
+ }
+
+ @Test
+ public void testLoadBalancingWithAllBadNodes() throws Exception {
+ Configuration conf = new Configuration();
+ KMSClientProvider p1 = mock(KMSClientProvider.class);
+ when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new IOException("p1"));
+ KMSClientProvider p2 = mock(KMSClientProvider.class);
+ when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new IOException("p2"));
+ KMSClientProvider p3 = mock(KMSClientProvider.class);
+ when(p3.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new IOException("p3"));
+ KMSClientProvider p4 = mock(KMSClientProvider.class);
+ when(p4.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new IOException("p4"));
+ when(p1.getKMSUrl()).thenReturn("p1");
+ when(p2.getKMSUrl()).thenReturn("p2");
+ when(p3.getKMSUrl()).thenReturn("p3");
+ when(p4.getKMSUrl()).thenReturn("p4");
+ KeyProvider kp = new LoadBalancingKMSClientProvider(
+ new KMSClientProvider[] { p1, p2, p3, p4 }, 0, conf);
+ try {
+ kp.createKey("test3", new Options(conf)).getName();
+ fail("Should fail since all providers threw an IOException");
+ } catch (Exception e) {
+ assertTrue(e instanceof IOException);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71385f9b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 70ba95f..c5a990b 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -24,9 +24,11 @@ import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
import org.apache.hadoop.crypto.key.KeyProvider.Options;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
+import org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.minikdc.MiniKdc;
@@ -99,6 +101,12 @@ public class TestKMS {
}
}
+ protected KeyProvider createProvider(URI uri, Configuration conf)
+ throws IOException {
+ return new LoadBalancingKMSClientProvider(
+ new KMSClientProvider[] { new KMSClientProvider(uri, conf) }, conf);
+ }
+
protected <T> T runServer(String keystore, String password, File confDir,
KMSCallable<T> callable) throws Exception {
return runServer(-1, keystore, password, confDir, callable);
@@ -305,7 +313,7 @@ public class TestKMS {
final URI uri = createKMSUri(getKMSUrl());
if (ssl) {
- KeyProvider testKp = new KMSClientProvider(uri, conf);
+ KeyProvider testKp = createProvider(uri, conf);
ThreadGroup threadGroup = Thread.currentThread().getThreadGroup();
while (threadGroup.getParent() != null) {
threadGroup = threadGroup.getParent();
@@ -335,12 +343,14 @@ public class TestKMS {
doAs(user, new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- final KeyProvider kp = new KMSClientProvider(uri, conf);
+ final KeyProvider kp = createProvider(uri, conf);
// getKeys() empty
Assert.assertTrue(kp.getKeys().isEmpty());
Thread.sleep(4000);
- Token<?>[] tokens = ((KMSClientProvider)kp).addDelegationTokens("myuser", new Credentials());
+ Token<?>[] tokens =
+ ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)kp)
+ .addDelegationTokens("myuser", new Credentials());
Assert.assertEquals(1, tokens.length);
Assert.assertEquals("kms-dt", tokens[0].getKind().toString());
return null;
@@ -348,12 +358,14 @@ public class TestKMS {
});
}
} else {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
// getKeys() empty
Assert.assertTrue(kp.getKeys().isEmpty());
Thread.sleep(4000);
- Token<?>[] tokens = ((KMSClientProvider)kp).addDelegationTokens("myuser", new Credentials());
+ Token<?>[] tokens =
+ ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)kp)
+ .addDelegationTokens("myuser", new Credentials());
Assert.assertEquals(1, tokens.length);
Assert.assertEquals("kms-dt", tokens[0].getKind().toString());
}
@@ -404,7 +416,7 @@ public class TestKMS {
Date started = new Date();
Configuration conf = new Configuration();
URI uri = createKMSUri(getKMSUrl());
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
// getKeys() empty
Assert.assertTrue(kp.getKeys().isEmpty());
@@ -687,7 +699,7 @@ public class TestKMS {
doAs("CREATE", new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
try {
Options options = new KeyProvider.Options(conf);
Map<String, String> attributes = options.getAttributes();
@@ -727,7 +739,7 @@ public class TestKMS {
doAs("DECRYPT_EEK", new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
try {
Options options = new KeyProvider.Options(conf);
Map<String, String> attributes = options.getAttributes();
@@ -760,7 +772,7 @@ public class TestKMS {
doAs("ROLLOVER", new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
try {
Options options = new KeyProvider.Options(conf);
Map<String, String> attributes = options.getAttributes();
@@ -804,7 +816,7 @@ public class TestKMS {
doAs("GET", new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
try {
Options options = new KeyProvider.Options(conf);
Map<String, String> attributes = options.getAttributes();
@@ -836,7 +848,7 @@ public class TestKMS {
final EncryptedKeyVersion ekv = doAs("GENERATE_EEK", new PrivilegedExceptionAction<EncryptedKeyVersion>() {
@Override
public EncryptedKeyVersion run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
try {
Options options = new KeyProvider.Options(conf);
Map<String, String> attributes = options.getAttributes();
@@ -861,7 +873,7 @@ public class TestKMS {
doAs("ROLLOVER", new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
try {
KeyProviderCryptoExtension kpce =
KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp);
@@ -891,7 +903,7 @@ public class TestKMS {
doAs("GENERATE_EEK", new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
try {
KeyProviderCryptoExtension kpce =
KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp);
@@ -964,7 +976,7 @@ public class TestKMS {
new PrivilegedExceptionAction<KeyProvider>() {
@Override
public KeyProvider run() throws Exception {
- KMSClientProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
kp.createKey("k1", new byte[16],
new KeyProvider.Options(conf));
return kp;
@@ -1041,7 +1053,7 @@ public class TestKMS {
new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KMSClientProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
kp.createKey("k0", new byte[16],
new KeyProvider.Options(conf));
@@ -1072,7 +1084,7 @@ public class TestKMS {
new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KMSClientProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
kp.createKey("k3", new byte[16],
new KeyProvider.Options(conf));
// Atleast 2 rollovers.. so should induce signer Exception
@@ -1132,7 +1144,7 @@ public class TestKMS {
doAs("client", new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
try {
kp.createKey("k", new KeyProvider.Options(conf));
Assert.fail();
@@ -1223,7 +1235,7 @@ public class TestKMS {
doAs("CREATE", new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
try {
KeyProvider.KeyVersion kv = kp.createKey("k0",
new KeyProvider.Options(conf));
@@ -1238,7 +1250,7 @@ public class TestKMS {
doAs("DELETE", new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
try {
kp.deleteKey("k0");
} catch (Exception ex) {
@@ -1251,7 +1263,7 @@ public class TestKMS {
doAs("SET_KEY_MATERIAL", new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
try {
KeyProvider.KeyVersion kv = kp.createKey("k1", new byte[16],
new KeyProvider.Options(conf));
@@ -1266,7 +1278,7 @@ public class TestKMS {
doAs("ROLLOVER", new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
try {
KeyProvider.KeyVersion kv = kp.rollNewVersion("k1");
Assert.assertNull(kv.getMaterial());
@@ -1280,7 +1292,7 @@ public class TestKMS {
doAs("SET_KEY_MATERIAL", new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
try {
KeyProvider.KeyVersion kv =
kp.rollNewVersion("k1", new byte[16]);
@@ -1296,7 +1308,7 @@ public class TestKMS {
doAs("GET", new PrivilegedExceptionAction<KeyVersion>() {
@Override
public KeyVersion run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
try {
kp.getKeyVersion("k1@0");
KeyVersion kv = kp.getCurrentKey("k1");
@@ -1313,7 +1325,7 @@ public class TestKMS {
new PrivilegedExceptionAction<EncryptedKeyVersion>() {
@Override
public EncryptedKeyVersion run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
try {
KeyProviderCryptoExtension kpCE = KeyProviderCryptoExtension.
createKeyProviderCryptoExtension(kp);
@@ -1330,7 +1342,7 @@ public class TestKMS {
doAs("DECRYPT_EEK", new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
try {
KeyProviderCryptoExtension kpCE = KeyProviderCryptoExtension.
createKeyProviderCryptoExtension(kp);
@@ -1345,7 +1357,7 @@ public class TestKMS {
doAs("GET_KEYS", new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
try {
kp.getKeys();
} catch (Exception ex) {
@@ -1358,7 +1370,7 @@ public class TestKMS {
doAs("GET_METADATA", new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
try {
kp.getMetadata("k1");
kp.getKeysMetadata("k1");
@@ -1385,7 +1397,7 @@ public class TestKMS {
@Override
public Void run() throws Exception {
try {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
KeyProvider.KeyVersion kv = kp.createKey("k2",
new KeyProvider.Options(conf));
Assert.fail();
@@ -1440,12 +1452,12 @@ public class TestKMS {
@Override
public Void run() throws Exception {
try {
- KMSClientProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
KeyProvider.KeyVersion kv = kp.createKey("ck0",
new KeyProvider.Options(conf));
EncryptedKeyVersion eek =
- kp.generateEncryptedKey("ck0");
- kp.decryptEncryptedKey(eek);
+ ((CryptoExtension)kp).generateEncryptedKey("ck0");
+ ((CryptoExtension)kp).decryptEncryptedKey(eek);
Assert.assertNull(kv.getMaterial());
} catch (Exception ex) {
Assert.fail(ex.getMessage());
@@ -1458,12 +1470,12 @@ public class TestKMS {
@Override
public Void run() throws Exception {
try {
- KMSClientProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
KeyProvider.KeyVersion kv = kp.createKey("ck1",
new KeyProvider.Options(conf));
EncryptedKeyVersion eek =
- kp.generateEncryptedKey("ck1");
- kp.decryptEncryptedKey(eek);
+ ((CryptoExtension)kp).generateEncryptedKey("ck1");
+ ((CryptoExtension)kp).decryptEncryptedKey(eek);
Assert.fail("admin user must not be allowed to decrypt !!");
} catch (Exception ex) {
}
@@ -1475,12 +1487,12 @@ public class TestKMS {
@Override
public Void run() throws Exception {
try {
- KMSClientProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
KeyProvider.KeyVersion kv = kp.createKey("ck2",
new KeyProvider.Options(conf));
EncryptedKeyVersion eek =
- kp.generateEncryptedKey("ck2");
- kp.decryptEncryptedKey(eek);
+ ((CryptoExtension)kp).generateEncryptedKey("ck2");
+ ((CryptoExtension)kp).decryptEncryptedKey(eek);
Assert.fail("admin user must not be allowed to decrypt !!");
} catch (Exception ex) {
}
@@ -1525,7 +1537,7 @@ public class TestKMS {
@Override
public Void run() throws Exception {
try {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
KeyProvider.KeyVersion kv = kp.createKey("ck0",
new KeyProvider.Options(conf));
Assert.assertNull(kv.getMaterial());
@@ -1540,7 +1552,7 @@ public class TestKMS {
@Override
public Void run() throws Exception {
try {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
KeyProvider.KeyVersion kv = kp.createKey("ck1",
new KeyProvider.Options(conf));
Assert.assertNull(kv.getMaterial());
@@ -1583,7 +1595,7 @@ public class TestKMS {
boolean caughtTimeout = false;
try {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
kp.getKeys();
} catch (SocketTimeoutException e) {
caughtTimeout = true;
@@ -1593,7 +1605,7 @@ public class TestKMS {
caughtTimeout = false;
try {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp)
.generateEncryptedKey("a");
} catch (SocketTimeoutException e) {
@@ -1604,7 +1616,7 @@ public class TestKMS {
caughtTimeout = false;
try {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp)
.decryptEncryptedKey(
new KMSClientProvider.KMSEncryptedKeyVersion("a",
@@ -1651,7 +1663,7 @@ public class TestKMS {
UserGroupInformation.getCurrentUser();
try {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
kp.createKey(keyA, new KeyProvider.Options(conf));
} catch (IOException ex) {
System.out.println(ex.getMessage());
@@ -1660,7 +1672,7 @@ public class TestKMS {
doAs("client", new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
KeyProviderDelegationTokenExtension kpdte =
KeyProviderDelegationTokenExtension.
createKeyProviderDelegationTokenExtension(kp);
@@ -1672,7 +1684,7 @@ public class TestKMS {
nonKerberosUgi.addCredentials(credentials);
try {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
kp.createKey(keyA, new KeyProvider.Options(conf));
} catch (IOException ex) {
System.out.println(ex.getMessage());
@@ -1681,7 +1693,7 @@ public class TestKMS {
nonKerberosUgi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
kp.createKey(keyD, new KeyProvider.Options(conf));
return null;
}
@@ -1767,7 +1779,7 @@ public class TestKMS {
new PrivilegedExceptionAction<KeyProvider>() {
@Override
public KeyProvider run() throws Exception {
- KMSClientProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
kp.createKey("k1", new byte[16],
new KeyProvider.Options(conf));
kp.createKey("k2", new byte[16],
@@ -1844,7 +1856,7 @@ public class TestKMS {
clientUgi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- final KeyProvider kp = new KMSClientProvider(uri, conf);
+ final KeyProvider kp = createProvider(uri, conf);
kp.createKey("kaa", new KeyProvider.Options(conf));
// authorized proxyuser
@@ -1956,7 +1968,7 @@ public class TestKMS {
fooUgi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
Assert.assertNotNull(kp.createKey("kaa",
new KeyProvider.Options(conf)));
return null;
@@ -1970,7 +1982,7 @@ public class TestKMS {
@Override
public Void run() throws Exception {
try {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
kp.createKey("kbb", new KeyProvider.Options(conf));
Assert.fail();
} catch (Exception ex) {
@@ -1986,7 +1998,7 @@ public class TestKMS {
barUgi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- KeyProvider kp = new KMSClientProvider(uri, conf);
+ KeyProvider kp = createProvider(uri, conf);
Assert.assertNotNull(kp.createKey("kcc",
new KeyProvider.Options(conf)));
return null;
[03/17] hadoop git commit: HDFS-7467. Provide storage tier
information for a directory via fsck. (Benoy Antony)
Posted by zj...@apache.org.
HDFS-7467. Provide storage tier information for a directory via fsck. (Benoy Antony)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d140d76a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d140d76a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d140d76a
Branch: refs/heads/YARN-2928
Commit: d140d76a43c88e326b9c2818578f22bd3563b969
Parents: caa42ad
Author: Benoy Antony <be...@apache.org>
Authored: Wed Feb 25 16:19:35 2015 -0800
Committer: Benoy Antony <be...@apache.org>
Committed: Wed Feb 25 16:19:35 2015 -0800
----------------------------------------------------------------------
.../hdfs/server/namenode/NamenodeFsck.java | 23 +-
.../server/namenode/StoragePolicySummary.java | 257 +++++++++++++++++++
.../org/apache/hadoop/hdfs/tools/DFSck.java | 2 +
.../hadoop/hdfs/server/namenode/TestFsck.java | 78 +++++-
.../namenode/TestStoragePolicySummary.java | 201 +++++++++++++++
5 files changed, 548 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d140d76a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index dc9494d..5134f3c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -25,6 +25,7 @@ import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
@@ -45,6 +46,7 @@ import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.RemotePeerFactory;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.net.Peer;
import org.apache.hadoop.hdfs.net.TcpPeerServer;
import org.apache.hadoop.hdfs.protocol.Block;
@@ -127,6 +129,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
private boolean showBlocks = false;
private boolean showLocations = false;
private boolean showRacks = false;
+ private boolean showStoragePolcies = false;
private boolean showprogress = false;
private boolean showCorruptFileBlocks = false;
@@ -164,6 +167,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
private List<String> snapshottableDirs = null;
private final BlockPlacementPolicy bpPolicy;
+ private StoragePolicySummary storageTypeSummary = null;
/**
* Filesystem checker.
@@ -200,6 +204,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
else if (key.equals("blocks")) { this.showBlocks = true; }
else if (key.equals("locations")) { this.showLocations = true; }
else if (key.equals("racks")) { this.showRacks = true; }
+ else if (key.equals("storagepolicies")) { this.showStoragePolcies = true; }
else if (key.equals("showprogress")) { this.showprogress = true; }
else if (key.equals("openforwrite")) {this.showOpenFiles = true; }
else if (key.equals("listcorruptfileblocks")) {
@@ -334,6 +339,11 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
return;
}
+ if (this.showStoragePolcies) {
+ storageTypeSummary = new StoragePolicySummary(
+ namenode.getNamesystem().getBlockManager().getStoragePolicies());
+ }
+
Result res = new Result(conf);
check(path, file, res);
@@ -342,6 +352,10 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
out.println(" Number of data-nodes:\t\t" + totalDatanodes);
out.println(" Number of racks:\t\t" + networktopology.getNumOfRacks());
+ if (this.showStoragePolcies) {
+ out.print(storageTypeSummary.toString());
+ }
+
out.println("FSCK ended at " + new Date() + " in "
+ (Time.now() - startTime + " milliseconds"));
@@ -492,7 +506,8 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
boolean isCorrupt = lBlk.isCorrupt();
String blkName = block.toString();
DatanodeInfo[] locs = lBlk.getLocations();
- NumberReplicas numberReplicas = namenode.getNamesystem().getBlockManager().countNodes(block.getLocalBlock());
+ NumberReplicas numberReplicas =
+ namenode.getNamesystem().getBlockManager().countNodes(block.getLocalBlock());
int liveReplicas = numberReplicas.liveReplicas();
res.totalReplicas += liveReplicas;
short targetFileReplication = file.getReplication();
@@ -501,6 +516,12 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
res.excessiveReplicas += (liveReplicas - targetFileReplication);
res.numOverReplicatedBlocks += 1;
}
+ //keep track of storage tier counts
+ if (this.showStoragePolcies && lBlk.getStorageTypes() != null) {
+ StorageType[] storageTypes = lBlk.getStorageTypes();
+ storageTypeSummary.add(Arrays.copyOf(storageTypes, storageTypes.length),
+ fsn.getBlockManager().getStoragePolicy(file.getStoragePolicy()));
+ }
// Check if block is Corrupt
if (isCorrupt) {
corrupt++;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d140d76a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySummary.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySummary.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySummary.java
new file mode 100644
index 0000000..bcdad35
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StoragePolicySummary.java
@@ -0,0 +1,257 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.text.NumberFormat;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumMap;
+import java.util.Formatter;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+
+/**
+ * Aggregate the storage type information for a set of blocks
+ *
+ */
+public class StoragePolicySummary {
+
+ Map<StorageTypeAllocation, Long> storageComboCounts = new HashMap<>();
+ final BlockStoragePolicy[] storagePolicies;
+ int totalBlocks;
+
+ StoragePolicySummary(BlockStoragePolicy[] storagePolicies) {
+ this.storagePolicies = storagePolicies;
+ }
+
+ // Add a storage type combination
+ void add(StorageType[] storageTypes, BlockStoragePolicy policy) {
+ StorageTypeAllocation storageCombo =
+ new StorageTypeAllocation(storageTypes, policy);
+ Long count = storageComboCounts.get(storageCombo);
+ if (count == null) {
+ storageComboCounts.put(storageCombo, 1l);
+ storageCombo.setActualStoragePolicy(
+ getStoragePolicy(storageCombo.getStorageTypes()));
+ } else {
+ storageComboCounts.put(storageCombo, count.longValue()+1);
+ }
+ totalBlocks++;
+ }
+
+ // sort the storageType combinations based on the total blocks counts
+ // in descending order
+ static List<Entry<StorageTypeAllocation, Long>> sortByComparator(
+ Map<StorageTypeAllocation, Long> unsortMap) {
+ List<Entry<StorageTypeAllocation, Long>> storageAllocations =
+ new LinkedList<>(unsortMap.entrySet());
+ // Sorting the list based on values
+ Collections.sort(storageAllocations,
+ new Comparator<Entry<StorageTypeAllocation, Long>>() {
+ public int compare(Entry<StorageTypeAllocation, Long> o1,
+ Entry<StorageTypeAllocation, Long> o2)
+ {
+ return o2.getValue().compareTo(o1.getValue());
+ }
+ });
+ return storageAllocations;
+ }
+
+ public String toString() {
+ StringBuilder compliantBlocksSB = new StringBuilder();
+ compliantBlocksSB.append("\nBlocks satisfying the specified storage policy:");
+ compliantBlocksSB.append("\nStorage Policy # of blocks % of blocks\n");
+ StringBuilder nonCompliantBlocksSB = new StringBuilder();
+ Formatter compliantFormatter = new Formatter(compliantBlocksSB);
+ Formatter nonCompliantFormatter = new Formatter(nonCompliantBlocksSB);
+ NumberFormat percentFormat = NumberFormat.getPercentInstance();
+ percentFormat.setMinimumFractionDigits(4);
+ percentFormat.setMaximumFractionDigits(4);
+ for (Map.Entry<StorageTypeAllocation, Long> storageComboCount:
+ sortByComparator(storageComboCounts)) {
+ double percent = (double) storageComboCount.getValue() /
+ (double) totalBlocks;
+ StorageTypeAllocation sta = storageComboCount.getKey();
+ if (sta.policyMatches()) {
+ compliantFormatter.format("%-25s %10d %20s%n",
+ sta.getStoragePolicyDescriptor(),
+ storageComboCount.getValue(),
+ percentFormat.format(percent));
+ } else {
+ if (nonCompliantBlocksSB.length() == 0) {
+ nonCompliantBlocksSB.append("\nBlocks NOT satisfying the specified storage policy:");
+ nonCompliantBlocksSB.append("\nStorage Policy ");
+ nonCompliantBlocksSB.append(
+ "Specified Storage Policy # of blocks % of blocks\n");
+ }
+ nonCompliantFormatter.format("%-35s %-20s %10d %20s%n",
+ sta.getStoragePolicyDescriptor(),
+ sta.getSpecifiedStoragePolicy().getName(),
+ storageComboCount.getValue(),
+ percentFormat.format(percent));
+ }
+ }
+ if (nonCompliantBlocksSB.length() == 0) {
+ nonCompliantBlocksSB.append("\nAll blocks satisfy specified storage policy.\n");
+ }
+ compliantFormatter.close();
+ nonCompliantFormatter.close();
+ return compliantBlocksSB.toString() + nonCompliantBlocksSB;
+ }
+
+ /**
+ *
+ * @param storageTypes - sorted array of storageTypes
+ * @return Storage Policy which matches the specific storage Combination
+ */
+ private BlockStoragePolicy getStoragePolicy(StorageType[] storageTypes) {
+ for (BlockStoragePolicy storagePolicy:storagePolicies) {
+ StorageType[] policyStorageTypes = storagePolicy.getStorageTypes();
+ policyStorageTypes = Arrays.copyOf(policyStorageTypes, policyStorageTypes.length);
+ Arrays.sort(policyStorageTypes);
+ if (policyStorageTypes.length <= storageTypes.length) {
+ int i = 0;
+ for (; i < policyStorageTypes.length; i++) {
+ if (policyStorageTypes[i] != storageTypes[i]) {
+ break;
+ }
+ }
+ if (i < policyStorageTypes.length) {
+ continue;
+ }
+ int j=policyStorageTypes.length;
+ for (; j < storageTypes.length; j++) {
+ if (policyStorageTypes[i-1] != storageTypes[j]) {
+ break;
+ }
+ }
+
+ if (j==storageTypes.length) {
+ return storagePolicy;
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Internal class which represents a unique Storage type combination
+ *
+ */
+ static class StorageTypeAllocation {
+ private final BlockStoragePolicy specifiedStoragePolicy;
+ private final StorageType[] storageTypes;
+ private BlockStoragePolicy actualStoragePolicy;
+
+ StorageTypeAllocation(StorageType[] storageTypes,
+ BlockStoragePolicy specifiedStoragePolicy) {
+ Arrays.sort(storageTypes);
+ this.storageTypes = storageTypes;
+ this.specifiedStoragePolicy = specifiedStoragePolicy;
+ }
+
+ StorageType[] getStorageTypes() {
+ return storageTypes;
+ }
+
+ BlockStoragePolicy getSpecifiedStoragePolicy() {
+ return specifiedStoragePolicy;
+ }
+
+ void setActualStoragePolicy(BlockStoragePolicy actualStoragePolicy) {
+ this.actualStoragePolicy = actualStoragePolicy;
+ }
+
+ BlockStoragePolicy getActualStoragePolicy() {
+ return actualStoragePolicy;
+ }
+
+ private static String getStorageAllocationAsString
+ (Map<StorageType, Integer> storageType_countmap) {
+ StringBuilder sb = new StringBuilder();
+ for (Map.Entry<StorageType, Integer>
+ storageTypeCountEntry:storageType_countmap.entrySet()) {
+ sb.append(storageTypeCountEntry.getKey().name()+ ":"
+ + storageTypeCountEntry.getValue() + ",");
+ }
+ if (sb.length() > 1) {
+ sb.deleteCharAt(sb.length()-1);
+ }
+ return sb.toString();
+ }
+
+ private String getStorageAllocationAsString() {
+ Map<StorageType, Integer> storageType_countmap =
+ new EnumMap<>(StorageType.class);
+ for (StorageType storageType: storageTypes) {
+ Integer count = storageType_countmap.get(storageType);
+ if (count == null) {
+ storageType_countmap.put(storageType, 1);
+ } else {
+ storageType_countmap.put(storageType, count.intValue()+1);
+ }
+ }
+ return (getStorageAllocationAsString(storageType_countmap));
+ }
+
+ String getStoragePolicyDescriptor() {
+ StringBuilder storagePolicyDescriptorSB = new StringBuilder();
+ if (actualStoragePolicy!=null) {
+ storagePolicyDescriptorSB.append(getStorageAllocationAsString())
+ .append("(")
+ .append(actualStoragePolicy.getName())
+ .append(")");
+ } else {
+ storagePolicyDescriptorSB.append(getStorageAllocationAsString());
+ }
+ return storagePolicyDescriptorSB.toString();
+ }
+
+ boolean policyMatches() {
+ return specifiedStoragePolicy.equals(actualStoragePolicy);
+ }
+
+ @Override
+ public String toString() {
+ return specifiedStoragePolicy.getName() + "|" + getStoragePolicyDescriptor();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(specifiedStoragePolicy,Arrays.hashCode(storageTypes));
+ }
+
+ @Override
+ public boolean equals(Object another) {
+ return (another instanceof StorageTypeAllocation &&
+ Objects.equals(specifiedStoragePolicy,
+ ((StorageTypeAllocation)another).specifiedStoragePolicy) &&
+ Arrays.equals(storageTypes,
+ ((StorageTypeAllocation)another).storageTypes));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d140d76a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
index 8c934c2..ec83a90 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
@@ -93,6 +93,7 @@ public class DFSck extends Configured implements Tool {
+ "\t-blocks\tprint out block report\n"
+ "\t-locations\tprint out locations for every block\n"
+ "\t-racks\tprint out network topology for data-node locations\n"
+ + "\t-storagepolicies\tprint out storage policy summary for the blocks\n"
+ "\t-showprogress\tshow progress in output. Default is OFF (no progress)\n"
+ "\t-blockId\tprint out which file this blockId belongs to, locations"
+ " (nodes, racks) of this block, and other diagnostics info"
@@ -266,6 +267,7 @@ public class DFSck extends Configured implements Tool {
else if (args[idx].equals("-blocks")) { url.append("&blocks=1"); }
else if (args[idx].equals("-locations")) { url.append("&locations=1"); }
else if (args[idx].equals("-racks")) { url.append("&racks=1"); }
+ else if (args[idx].equals("-storagepolicies")) { url.append("&storagepolicies=1"); }
else if (args[idx].equals("-showprogress")) { url.append("&showprogress=1"); }
else if (args[idx].equals("-list-corruptfileblocks")) {
url.append("&listcorruptfileblocks=1");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d140d76a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index 5574a30..35a5a03 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -18,6 +18,18 @@
package org.apache.hadoop.hdfs.server.namenode;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -41,7 +53,6 @@ import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import com.google.common.collect.Sets;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -58,6 +69,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -83,17 +95,7 @@ import org.apache.log4j.PatternLayout;
import org.apache.log4j.RollingFileAppender;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Matchers.anyBoolean;
-import static org.mockito.Matchers.anyLong;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
+import com.google.common.collect.Sets;
/**
* A JUnit test for doing fsck
@@ -1326,4 +1328,56 @@ public class TestFsck {
}
}
}
+
+ private void writeFile(final DistributedFileSystem dfs,
+ Path dir, String fileName) throws IOException {
+ Path filePath = new Path(dir.toString() + Path.SEPARATOR + fileName);
+ final FSDataOutputStream out = dfs.create(filePath);
+ out.writeChars("teststring");
+ out.close();
+ }
+
+ private void writeFile(final DistributedFileSystem dfs,
+ String dirName, String fileName, String StoragePolicy) throws IOException {
+ Path dirPath = new Path(dirName);
+ dfs.mkdirs(dirPath);
+ dfs.setStoragePolicy(dirPath, StoragePolicy);
+ writeFile(dfs, dirPath, fileName);
+ }
+
+ /**
+ * Test storage policy display
+ */
+ @Test
+ public void testStoragePoliciesCK() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(3)
+ .storageTypes(
+ new StorageType[] {StorageType.DISK, StorageType.ARCHIVE})
+ .build();
+ try {
+ cluster.waitActive();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ writeFile(dfs, "/testhot", "file", "HOT");
+ writeFile(dfs, "/testwarm", "file", "WARM");
+ writeFile(dfs, "/testcold", "file", "COLD");
+ String outStr = runFsck(conf, 0, true, "/", "-storagepolicies");
+ assertTrue(outStr.contains("DISK:3(HOT)"));
+ assertTrue(outStr.contains("DISK:1,ARCHIVE:2(WARM)"));
+ assertTrue(outStr.contains("ARCHIVE:3(COLD)"));
+ assertTrue(outStr.contains("All blocks satisfy specified storage policy."));
+ dfs.setStoragePolicy(new Path("/testhot"), "COLD");
+ dfs.setStoragePolicy(new Path("/testwarm"), "COLD");
+ outStr = runFsck(conf, 0, true, "/", "-storagepolicies");
+ assertTrue(outStr.contains("DISK:3(HOT)"));
+ assertTrue(outStr.contains("DISK:1,ARCHIVE:2(WARM)"));
+ assertTrue(outStr.contains("ARCHIVE:3(COLD)"));
+ assertFalse(outStr.contains("All blocks satisfy specified storage policy."));
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d140d76a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySummary.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySummary.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySummary.java
new file mode 100644
index 0000000..60c9318
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySummary.java
@@ -0,0 +1,201 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
+import org.apache.hadoop.hdfs.server.namenode.StoragePolicySummary.StorageTypeAllocation;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestStoragePolicySummary {
+
+ private Map<String, Long> convertToStringMap(StoragePolicySummary sts) {
+ LinkedHashMap<String, Long> actualOutput = new LinkedHashMap<>();
+ for (Map.Entry<StorageTypeAllocation, Long> entry:
+ StoragePolicySummary.sortByComparator(sts.storageComboCounts)) {
+ actualOutput.put(entry.getKey().toString(), entry.getValue());
+ }
+ return actualOutput;
+ }
+
+ @Test
+ public void testMultipleHots() {
+ BlockStoragePolicySuite bsps = BlockStoragePolicySuite.createDefaultSuite();
+ StoragePolicySummary sts = new StoragePolicySummary(bsps.getAllPolicies());
+ BlockStoragePolicy hot = bsps.getPolicy("HOT");
+ sts.add(new StorageType[]{StorageType.DISK},hot);
+ sts.add(new StorageType[]{StorageType.DISK,StorageType.DISK},hot);
+ sts.add(new StorageType[]{StorageType.DISK,
+ StorageType.DISK,StorageType.DISK},hot);
+ sts.add(new StorageType[]{StorageType.DISK,
+ StorageType.DISK,StorageType.DISK,StorageType.DISK},hot);
+ Map<String, Long> actualOutput = convertToStringMap(sts);
+ Assert.assertEquals(4,actualOutput.size());
+ Map<String, Long> expectedOutput = new HashMap<>();
+ expectedOutput.put("HOT|DISK:1(HOT)", 1l);
+ expectedOutput.put("HOT|DISK:2(HOT)", 1l);
+ expectedOutput.put("HOT|DISK:3(HOT)", 1l);
+ expectedOutput.put("HOT|DISK:4(HOT)", 1l);
+ Assert.assertEquals(expectedOutput,actualOutput);
+ }
+
+ @Test
+ public void testMultipleHotsWithDifferentCounts() {
+ BlockStoragePolicySuite bsps = BlockStoragePolicySuite.createDefaultSuite();
+ StoragePolicySummary sts = new StoragePolicySummary(bsps.getAllPolicies());
+ BlockStoragePolicy hot = bsps.getPolicy("HOT");
+ sts.add(new StorageType[]{StorageType.DISK},hot);
+ sts.add(new StorageType[]{StorageType.DISK,StorageType.DISK},hot);
+ sts.add(new StorageType[]{StorageType.DISK,StorageType.DISK},hot);
+ sts.add(new StorageType[]{StorageType.DISK,
+ StorageType.DISK,StorageType.DISK},hot);
+ sts.add(new StorageType[]{StorageType.DISK,
+ StorageType.DISK,StorageType.DISK},hot);
+ sts.add(new StorageType[]{StorageType.DISK,
+ StorageType.DISK,StorageType.DISK,StorageType.DISK},hot);
+ Map<String, Long> actualOutput = convertToStringMap(sts);
+ Assert.assertEquals(4,actualOutput.size());
+ Map<String, Long> expectedOutput = new HashMap<>();
+ expectedOutput.put("HOT|DISK:1(HOT)", 1l);
+ expectedOutput.put("HOT|DISK:2(HOT)", 2l);
+ expectedOutput.put("HOT|DISK:3(HOT)", 2l);
+ expectedOutput.put("HOT|DISK:4(HOT)", 1l);
+ Assert.assertEquals(expectedOutput,actualOutput);
+ }
+
+ @Test
+ public void testMultipleWarmsInDifferentOrder() {
+ BlockStoragePolicySuite bsps = BlockStoragePolicySuite.createDefaultSuite();
+ StoragePolicySummary sts = new StoragePolicySummary(bsps.getAllPolicies());
+ BlockStoragePolicy warm = bsps.getPolicy("WARM");
+ //DISK:1,ARCHIVE:1
+ sts.add(new StorageType[]{StorageType.DISK,StorageType.ARCHIVE},warm);
+ sts.add(new StorageType[]{StorageType.ARCHIVE,StorageType.DISK},warm);
+ //DISK:2,ARCHIVE:1
+ sts.add(new StorageType[]{StorageType.ARCHIVE,
+ StorageType.DISK,StorageType.DISK},warm);
+ sts.add(new StorageType[]{StorageType.DISK,
+ StorageType.ARCHIVE,StorageType.DISK},warm);
+ sts.add(new StorageType[]{StorageType.DISK,
+ StorageType.DISK,StorageType.ARCHIVE},warm);
+ //DISK:1,ARCHIVE:2
+ sts.add(new StorageType[]{StorageType.DISK,
+ StorageType.ARCHIVE,StorageType.ARCHIVE},warm);
+ sts.add(new StorageType[]{StorageType.ARCHIVE,
+ StorageType.DISK,StorageType.ARCHIVE},warm);
+ sts.add(new StorageType[]{StorageType.ARCHIVE,
+ StorageType.ARCHIVE,StorageType.DISK},warm);
+ //DISK:2,ARCHIVE:2
+ sts.add(new StorageType[]{StorageType.ARCHIVE,
+ StorageType.ARCHIVE,StorageType.DISK,StorageType.DISK},warm);
+ Map<String, Long> actualOutput = convertToStringMap(sts);
+ Assert.assertEquals(4,actualOutput.size());
+ Map<String, Long> expectedOutput = new HashMap<>();
+ expectedOutput.put("WARM|DISK:1,ARCHIVE:1(WARM)", 2l);
+ expectedOutput.put("WARM|DISK:2,ARCHIVE:1", 3l);
+ expectedOutput.put("WARM|DISK:1,ARCHIVE:2(WARM)", 3l);
+ expectedOutput.put("WARM|DISK:2,ARCHIVE:2", 1l);
+ Assert.assertEquals(expectedOutput,actualOutput);
+ }
+
+ @Test
+ public void testDifferentSpecifiedPolicies() {
+ BlockStoragePolicySuite bsps = BlockStoragePolicySuite.createDefaultSuite();
+ StoragePolicySummary sts = new StoragePolicySummary(bsps.getAllPolicies());
+ BlockStoragePolicy hot = bsps.getPolicy("HOT");
+ BlockStoragePolicy warm = bsps.getPolicy("WARM");
+ BlockStoragePolicy cold = bsps.getPolicy("COLD");
+ //DISK:3
+ sts.add(new StorageType[]{StorageType.DISK,StorageType.DISK,StorageType.DISK},hot);
+ sts.add(new StorageType[]{StorageType.DISK,StorageType.DISK,StorageType.DISK},hot);
+ sts.add(new StorageType[]{StorageType.DISK,StorageType.DISK,StorageType.DISK},warm);
+ sts.add(new StorageType[]{StorageType.DISK,StorageType.DISK,StorageType.DISK},cold);
+ //DISK:1,ARCHIVE:2
+ sts.add(new StorageType[]{StorageType.DISK,
+ StorageType.ARCHIVE,StorageType.ARCHIVE},hot);
+ sts.add(new StorageType[]{StorageType.ARCHIVE,
+ StorageType.DISK,StorageType.ARCHIVE},warm);
+ sts.add(new StorageType[]{StorageType.ARCHIVE,
+ StorageType.ARCHIVE,StorageType.DISK},cold);
+ sts.add(new StorageType[]{StorageType.ARCHIVE,
+ StorageType.ARCHIVE,StorageType.DISK},cold);
+ //ARCHIVE:3
+ sts.add(new StorageType[]{StorageType.ARCHIVE,
+ StorageType.ARCHIVE,StorageType.ARCHIVE},hot);
+ sts.add(new StorageType[]{StorageType.ARCHIVE,
+ StorageType.ARCHIVE,StorageType.ARCHIVE},hot);
+ sts.add(new StorageType[]{StorageType.ARCHIVE,
+ StorageType.ARCHIVE,StorageType.ARCHIVE},warm);
+ sts.add(new StorageType[]{StorageType.ARCHIVE,
+ StorageType.ARCHIVE,StorageType.ARCHIVE},cold);
+ Map<String, Long> actualOutput = convertToStringMap(sts);
+ Assert.assertEquals(9,actualOutput.size());
+ Map<String, Long> expectedOutput = new HashMap<>();
+ expectedOutput.put("HOT|DISK:3(HOT)", 2l);
+ expectedOutput.put("COLD|DISK:1,ARCHIVE:2(WARM)", 2l);
+ expectedOutput.put("HOT|ARCHIVE:3(COLD)", 2l);
+ expectedOutput.put("WARM|DISK:3(HOT)", 1l);
+ expectedOutput.put("COLD|DISK:3(HOT)", 1l);
+ expectedOutput.put("WARM|ARCHIVE:3(COLD)", 1l);
+ expectedOutput.put("WARM|DISK:1,ARCHIVE:2(WARM)", 1l);
+ expectedOutput.put("COLD|ARCHIVE:3(COLD)", 1l);
+ expectedOutput.put("HOT|DISK:1,ARCHIVE:2(WARM)", 1l);
+ Assert.assertEquals(expectedOutput,actualOutput);
+ }
+
+ @Test
+ public void testSortInDescendingOrder() {
+ BlockStoragePolicySuite bsps = BlockStoragePolicySuite.createDefaultSuite();
+ StoragePolicySummary sts = new StoragePolicySummary(bsps.getAllPolicies());
+ BlockStoragePolicy hot = bsps.getPolicy("HOT");
+ BlockStoragePolicy warm = bsps.getPolicy("WARM");
+ BlockStoragePolicy cold = bsps.getPolicy("COLD");
+ //DISK:3
+ sts.add(new StorageType[]{StorageType.DISK,StorageType.DISK,StorageType.DISK},hot);
+ sts.add(new StorageType[]{StorageType.DISK,StorageType.DISK,StorageType.DISK},hot);
+ //DISK:1,ARCHIVE:2
+ sts.add(new StorageType[]{StorageType.DISK,
+ StorageType.ARCHIVE,StorageType.ARCHIVE},warm);
+ sts.add(new StorageType[]{StorageType.ARCHIVE,
+ StorageType.DISK,StorageType.ARCHIVE},warm);
+ sts.add(new StorageType[]{StorageType.ARCHIVE,
+ StorageType.ARCHIVE,StorageType.DISK},warm);
+ //ARCHIVE:3
+ sts.add(new StorageType[]{StorageType.ARCHIVE,
+ StorageType.ARCHIVE,StorageType.ARCHIVE},cold);
+ sts.add(new StorageType[]{StorageType.ARCHIVE,
+ StorageType.ARCHIVE,StorageType.ARCHIVE},cold);
+ sts.add(new StorageType[]{StorageType.ARCHIVE,
+ StorageType.ARCHIVE,StorageType.ARCHIVE},cold);
+ sts.add(new StorageType[]{StorageType.ARCHIVE,
+ StorageType.ARCHIVE,StorageType.ARCHIVE},cold);
+ Map<String, Long> actualOutput = convertToStringMap(sts);
+ Assert.assertEquals(3,actualOutput.size());
+ Map<String, Long> expectedOutput = new LinkedHashMap<>();
+ expectedOutput.put("COLD|ARCHIVE:3(COLD)", 4l);
+ expectedOutput.put("WARM|DISK:1,ARCHIVE:2(WARM)", 3l);
+ expectedOutput.put("HOT|DISK:3(HOT)", 2l);
+ Assert.assertEquals(expectedOutput.toString(),actualOutput.toString());
+ }
+}
[13/17] hadoop git commit: HADOOP-9922. hadoop windows native build
will fail in 32 bit machine. Contributed by Kiran Kumar M R.
Posted by zj...@apache.org.
HADOOP-9922. hadoop windows native build will fail in 32 bit machine. Contributed by Kiran Kumar M R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2214dab6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2214dab6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2214dab6
Branch: refs/heads/YARN-2928
Commit: 2214dab60ff11b8de74c9d661585452a078fe0c1
Parents: f0c980a
Author: cnauroth <cn...@apache.org>
Authored: Thu Feb 26 12:41:33 2015 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Thu Feb 26 12:41:33 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../hadoop-common/src/main/native/native.sln | 8 +--
.../src/main/native/native.vcxproj | 40 ++++++++++++
.../src/main/winutils/include/winutils.h | 5 +-
.../src/main/winutils/libwinutils.c | 2 +-
.../src/main/winutils/libwinutils.vcxproj | 64 +++++++++++++++++++-
.../hadoop-common/src/main/winutils/service.c | 8 +--
.../hadoop-common/src/main/winutils/task.c | 2 +-
.../src/main/winutils/winutils.sln | 10 +++
.../src/main/winutils/winutils.vcxproj | 61 ++++++++++++++++++-
10 files changed, 189 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2214dab6/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index ca27463..1d9a6d4 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1010,6 +1010,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11629. WASB filesystem should not start BandwidthGaugeUpdater if
fs.azure.skip.metrics set to true. (Shanyu Zhao via cnauroth)
+ HADOOP-9922. hadoop windows native build will fail in 32 bit machine.
+ (Kiran Kumar M R via cnauroth)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2214dab6/hadoop-common-project/hadoop-common/src/main/native/native.sln
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/native.sln b/hadoop-common-project/hadoop-common/src/main/native/native.sln
index 40a7821..54bc17e 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/native.sln
+++ b/hadoop-common-project/hadoop-common/src/main/native/native.sln
@@ -31,14 +31,14 @@ Global
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|Mixed Platforms.ActiveCfg = Release|x64
{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|Mixed Platforms.Build.0 = Release|x64
- {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|Win32.ActiveCfg = Release|x64
- {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|Win32.Build.0 = Release|x64
+ {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|Win32.ActiveCfg = Release|Win32
+ {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|Win32.Build.0 = Release|Win32
{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|x64.ActiveCfg = Release|x64
{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|x64.Build.0 = Release|x64
{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|Mixed Platforms.ActiveCfg = Release|x64
{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|Mixed Platforms.Build.0 = Release|x64
- {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|Win32.ActiveCfg = Release|x64
- {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|Win32.Build.0 = Release|x64
+ {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|Win32.ActiveCfg = Release|Win32
+ {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|Win32.Build.0 = Release|Win32
{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|x64.ActiveCfg = Release|x64
{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|x64.Build.0 = Release|x64
EndGlobalSection
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2214dab6/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj b/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj
index 2d60e56..0912c6a 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj
+++ b/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj
@@ -19,6 +19,10 @@
<Project DefaultTargets="CheckRequireSnappy;Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
@@ -30,6 +34,12 @@
<RootNamespace>native</RootNamespace>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <ConfigurationType>DynamicLibrary</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
@@ -39,10 +49,19 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <LinkIncremental>false</LinkIncremental>
+ <OutDir>..\..\..\target\bin\</OutDir>
+ <IntDir>..\..\..\target\native\$(Configuration)\</IntDir>
+ <TargetName>hadoop</TargetName>
+ </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<OutDir>..\..\..\target\bin\</OutDir>
@@ -65,6 +84,27 @@
Text="Required snappy library could not be found. SnappyLibrary=$(SnappyLibrary), SnappyInclude=$(SnappyInclude), CustomSnappyLib=$(CustomSnappyLib), CustomSnappyInclude=$(CustomSnappyInclude), CustomSnappyPrefix=$(CustomSnappyPrefix)"
Condition="'$(RequireSnappy)' == 'true' And '$(SnappyEnabled)' != 'true'" />
</Target>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <ClCompile>
+ <WarningLevel>Level3</WarningLevel>
+ <PrecompiledHeader>NotUsing</PrecompiledHeader>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_WINDOWS;_USRDLL;NATIVE_EXPORTS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <AdditionalIncludeDirectories>..\winutils\include;..\..\..\target\native\javah;%JAVA_HOME%\include;%JAVA_HOME%\include\win32;.\src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+ <CompileAs>CompileAsC</CompileAs>
+ <DisableSpecificWarnings>4244</DisableSpecificWarnings>
+ </ClCompile>
+ <Link>
+ <SubSystem>Windows</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ <AdditionalDependencies>Ws2_32.lib;libwinutils.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ <AdditionalLibraryDirectories>..\..\..\target\bin;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
+ </Link>
+ </ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2214dab6/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h b/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
index 0ac9adb..f72802c 100644
--- a/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
+++ b/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
@@ -248,8 +248,9 @@ DWORD BuildServiceSecurityDescriptor(
__out PSECURITY_DESCRIPTOR* pSD);
DWORD AddNodeManagerAndUserACEsToObject(
- __in HANDLE hProcess,
- __in LPWSTR user);
+ __in HANDLE hObject,
+ __in LPWSTR user,
+ __in ACCESS_MASK accessMask);
DWORD GetSecureJobObjectName(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2214dab6/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c b/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
index 5e775df..98fe3ab 100644
--- a/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
+++ b/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
@@ -2788,7 +2788,7 @@ DWORD BuildServiceSecurityDescriptor(
__out PSECURITY_DESCRIPTOR* pSD) {
DWORD dwError = ERROR_SUCCESS;
- int crt = 0;
+ unsigned int crt = 0;
int len = 0;
EXPLICIT_ACCESS* eas = NULL;
LPWSTR lpszSD = NULL;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2214dab6/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj b/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj
index 37b7f31..3c958e0 100644
--- a/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj
+++ b/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj
@@ -19,10 +19,18 @@
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
@@ -39,21 +47,38 @@
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+ <ConfigurationType>StaticLibrary</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <ConfigurationType>StaticLibrary</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup>
<IncludePath>include;$(IncludePath)</IncludePath>
@@ -61,9 +86,15 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
</PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <LinkIncremental>true</LinkIncremental>
+ </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
</PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <LinkIncremental>false</LinkIncremental>
+ </PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
@@ -77,6 +108,19 @@
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_DEBUG;_UNICODE;UNICODE;WSCE_CONFIG_DIR=$(WsceConfigDir);WSCE_CONFIG_FILE=$(WsceConfigFile);%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
@@ -95,13 +139,31 @@
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <ClCompile>
+ <WarningLevel>Level3</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <!-- <Optimization>MaxSpeed</Optimization> -->
+ <Optimization>Disabled</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;NDEBUG;_UNICODE;UNICODE;WSCE_CONFIG_DIR=$(WsceConfigDir);WSCE_CONFIG_FILE=$(WsceConfigFile);%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
<ItemDefinitionGroup>
<ClCompile>
<AdditionalIncludeDirectories>$(IntermediateOutputPath)</AdditionalIncludeDirectories>
</ClCompile>
<Midl>
<ApplicationConfigurationMode>true</ApplicationConfigurationMode>
- <TargetEnvironment>X64</TargetEnvironment>
+ <TargetEnvironment>$(Platform)</TargetEnvironment>
<OutputDirectory>$(IntermediateOutputPath)</OutputDirectory>
<GenerateStublessProxies>true</GenerateStublessProxies>
<ValidateAllParameters>true</ValidateAllParameters>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2214dab6/hadoop-common-project/hadoop-common/src/main/winutils/service.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/service.c b/hadoop-common-project/hadoop-common/src/main/winutils/service.c
index a0f8a66..ba35003 100644
--- a/hadoop-common-project/hadoop-common/src/main/winutils/service.c
+++ b/hadoop-common-project/hadoop-common/src/main/winutils/service.c
@@ -182,7 +182,7 @@ BOOL IsSidInList(
__in size_t cAllowedSids,
__in_ecount(cAllowedSids) PSID* allowedSids) {
- int crtSid = 0;
+ size_t crtSid = 0;
for (crtSid = 0; crtSid < cAllowedSids; ++crtSid) {
if (EqualSid(trustee, allowedSids[crtSid])) {
@@ -206,7 +206,7 @@ DWORD ValidateConfigurationFile() {
BOOL daclPresent = FALSE;
BOOL daclDefaulted = FALSE;
PACL pDacl = NULL;
- int crt = 0, crtSid = 0;
+ unsigned int crt = 0, crtSid = 0;
WELL_KNOWN_SID_TYPE allowedSidTypes[] = {
WinLocalSystemSid,
WinBuiltinAdministratorsSid};
@@ -350,7 +350,7 @@ DWORD InitLocalDirs() {
DWORD dwError = ERROR_SUCCESS;
size_t len = 0;
LPCWSTR value = NULL;
- int crt = 0;
+ size_t crt = 0;
dwError = GetConfigValue(
@@ -396,7 +396,7 @@ done:
DWORD ValidateLocalPath(LPCWSTR lpszPath) {
DWORD dwError = ERROR_SUCCESS;
int compareResult = 0;
- int crt = 0;
+ unsigned int crt = 0;
int cchLocalBuffer = 0;
WCHAR localBuffer[MAX_PATH+1];
BOOLEAN nullFound = FALSE;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2214dab6/hadoop-common-project/hadoop-common/src/main/winutils/task.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/task.c b/hadoop-common-project/hadoop-common/src/main/winutils/task.c
index bfdbd63..21b1893 100644
--- a/hadoop-common-project/hadoop-common/src/main/winutils/task.c
+++ b/hadoop-common-project/hadoop-common/src/main/winutils/task.c
@@ -129,7 +129,7 @@ DWORD BuildImpersonateSecurityDescriptor(__out PSECURITY_DESCRIPTOR* ppSD) {
WCHAR** tokens = NULL;
size_t len = 0;
size_t count = 0;
- int crt = 0;
+ size_t crt = 0;
PSECURITY_DESCRIPTOR pSD = NULL;
dwError = GetConfigValue(wsceConfigRelativePath, NM_WSCE_IMPERSONATE_ALLOWED, &len, &value);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2214dab6/hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln b/hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln
index d2784b8..876e66d 100644
--- a/hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln
+++ b/hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln
@@ -26,16 +26,26 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libwinutils", "libwinutils.
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
+ Debug|Win32 = Debug|Win32
Debug|x64 = Debug|x64
+ Release|Win32 = Release|Win32
Release|x64 = Release|x64
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
+ {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|Win32.ActiveCfg = Release|Win32
+ {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|Win32.Build.0 = Release|Win32
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.ActiveCfg = Release|x64
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.Build.0 = Release|x64
+ {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|Win32.ActiveCfg = Release|Win32
+ {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|Win32.Build.0 = Release|Win32
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|x64.ActiveCfg = Release|x64
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|x64.Build.0 = Release|x64
+ {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|Win32.ActiveCfg = Release|Win32
+ {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|Win32.Build.0 = Release|Win32
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.ActiveCfg = Release|x64
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.Build.0 = Release|x64
+ {12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|Win32.ActiveCfg = Release|Win32
+ {12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|Win32.Build.0 = Release|Win32
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|x64.ActiveCfg = Release|x64
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|x64.Build.0 = Release|x64
EndGlobalSection
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2214dab6/hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj b/hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj
index d736084..9ecba0a 100644
--- a/hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj
+++ b/hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj
@@ -19,10 +19,18 @@
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
@@ -39,6 +47,11 @@
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>true</UseDebugLibraries>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
@@ -46,14 +59,26 @@
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <UseDebugLibraries>false</UseDebugLibraries>
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
+ <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup>
<IncludePath>include;$(IncludePath)</IncludePath>
@@ -61,6 +86,9 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
</PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <LinkIncremental>false</LinkIncremental>
+ </PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
@@ -74,6 +102,19 @@
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <ClCompile>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <WarningLevel>Level4</WarningLevel>
+ <Optimization>Disabled</Optimization>
+ <PreprocessorDefinitions>WIN32;_CONSOLE;_DEBUG;_UNICODE;UNICODE;WSCE_CONFIG_DIR=$(WsceConfigDir);WSCE_CONFIG_FILE=$(WsceConfigFile);%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ </Link>
+ </ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
@@ -92,6 +133,24 @@
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ <ClCompile>
+ <WarningLevel>Level3</WarningLevel>
+ <PrecompiledHeader>
+ </PrecompiledHeader>
+ <!-- <Optimization>MaxSpeed</Optimization> -->
+ <Optimization>Disabled</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <PreprocessorDefinitions>WIN32;_CONSOLE;NDEBUG;_UNICODE;UNICODE;WSCE_CONFIG_DIR=$(WsceConfigDir);WSCE_CONFIG_FILE=$(WsceConfigFile);%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ <OptimizeReferences>true</OptimizeReferences>
+ </Link>
+ </ItemDefinitionGroup>
<ItemDefinitionGroup>
<ClCompile>
<AdditionalIncludeDirectories>$(IntermediateOutputPath)</AdditionalIncludeDirectories>
@@ -103,7 +162,7 @@
</CustomBuildStep>
<Midl>
<ApplicationConfigurationMode>true</ApplicationConfigurationMode>
- <TargetEnvironment>X64</TargetEnvironment>
+ <TargetEnvironment>$(Platform)</TargetEnvironment>
<OutputDirectory>$(IntermediateOutputPath)</OutputDirectory>
<GenerateStublessProxies>true</GenerateStublessProxies>
<ValidateAllParameters>true</ValidateAllParameters>
[09/17] hadoop git commit: YARN-3256.
TestClientToAMTokens#testClientTokenRace is not running against all
Schedulers even when using ParameterizedSchedulerTestBase. Contributed by
Anubhav Dhoot.
Posted by zj...@apache.org.
YARN-3256. TestClientToAMTokens#testClientTokenRace is not running against
all Schedulers even when using ParameterizedSchedulerTestBase. Contributed
by Anubhav Dhoot.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d4296f0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d4296f0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d4296f0
Branch: refs/heads/YARN-2928
Commit: 0d4296f0e0f545267f2e39a868d4ffefc9844db8
Parents: 166eecf
Author: Devaraj K <de...@apache.org>
Authored: Thu Feb 26 15:45:41 2015 +0530
Committer: Devaraj K <de...@apache.org>
Committed: Thu Feb 26 15:45:41 2015 +0530
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 4 ++++
.../server/resourcemanager/security/TestClientToAMTokens.java | 3 ---
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d4296f0/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e5148eb..ac3cbb2 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -658,6 +658,10 @@ Release 2.7.0 - UNRELEASED
YARN-3239. WebAppProxy does not support a final tracking url which has
query fragments and params (Jian He via jlowe)
+ YARN-3256. TestClientToAMTokens#testClientTokenRace is not running against
+ all Schedulers even when using ParameterizedSchedulerTestBase.
+ (Anubhav Dhoot via devaraj)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d4296f0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
index 78bc728..499b4d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
@@ -33,7 +33,6 @@ import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.security.PrivilegedAction;
import java.security.PrivilegedExceptionAction;
-import java.util.Arrays;
import java.util.Timer;
import java.util.TimerTask;
@@ -43,7 +42,6 @@ import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.Server;
@@ -421,7 +419,6 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
@Test(timeout=20000)
public void testClientTokenRace() throws Exception {
- final Configuration conf = new Configuration();
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
UserGroupInformation.setConfiguration(conf);
[17/17] hadoop git commit: Merge remote-tracking branch
'apache/trunk' into YARN-2928
Posted by zj...@apache.org.
Merge remote-tracking branch 'apache/trunk' into YARN-2928
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7788f977
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7788f977
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7788f977
Branch: refs/heads/YARN-2928
Commit: 7788f9773aab9c312b75834209d5debeda9d2820
Parents: 6bb198e bfbf076
Author: Zhijie Shen <zj...@apache.org>
Authored: Thu Feb 26 15:10:08 2015 -0800
Committer: Zhijie Shen <zj...@apache.org>
Committed: Thu Feb 26 15:10:08 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 11 +
.../src/main/bin/hadoop-functions.sh | 2 +-
.../crypto/key/kms/KMSClientProvider.java | 84 ++++-
.../key/kms/LoadBalancingKMSClientProvider.java | 347 +++++++++++++++++++
.../hadoop-common/src/main/native/native.sln | 8 +-
.../src/main/native/native.vcxproj | 40 +++
.../src/main/winutils/include/winutils.h | 5 +-
.../src/main/winutils/libwinutils.c | 2 +-
.../src/main/winutils/libwinutils.vcxproj | 64 +++-
.../hadoop-common/src/main/winutils/service.c | 8 +-
.../hadoop-common/src/main/winutils/task.c | 2 +-
.../src/main/winutils/winutils.sln | 10 +
.../src/main/winutils/winutils.vcxproj | 61 +++-
.../src/site/markdown/RackAwareness.md | 5 +-
.../kms/TestLoadBalancingKMSClientProvider.java | 166 +++++++++
.../hadoop-kms/src/main/conf/kms-env.sh | 2 +-
.../hadoop-kms/src/main/libexec/kms-config.sh | 2 +-
.../hadoop-kms/src/main/sbin/kms.sh | 2 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 114 +++---
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 2 +-
.../src/main/conf/httpfs-env.sh | 53 ++-
.../src/main/libexec/httpfs-config.sh | 206 +++--------
.../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 114 ++++--
.../src/main/tomcat/ssl-server.xml | 135 --------
.../src/main/tomcat/ssl-server.xml.conf | 135 ++++++++
.../src/site/apt/ServerSetup.apt.vm | 159 ---------
.../src/site/apt/UsingHttpTools.apt.vm | 87 -----
.../src/site/apt/index.apt.vm | 83 -----
.../src/site/markdown/ServerSetup.md.vm | 121 +++++++
.../src/site/markdown/UsingHttpTools.md | 62 ++++
.../src/site/markdown/index.md | 52 +++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 20 ++
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 5 +-
.../org/apache/hadoop/hdfs/DFSInputStream.java | 33 +-
.../hdfs/server/datanode/DirectoryScanner.java | 26 +-
.../hdfs/server/namenode/FSNamesystem.java | 3 +
.../hdfs/server/namenode/NamenodeFsck.java | 59 +++-
.../server/namenode/StoragePolicySummary.java | 257 ++++++++++++++
.../org/apache/hadoop/hdfs/tools/DFSck.java | 2 +
.../src/main/native/libhdfs/os/windows/thread.c | 2 +-
.../libhdfs/os/windows/thread_local_storage.c | 8 +
.../src/main/webapps/hdfs/explorer.html | 2 +
.../src/main/webapps/hdfs/explorer.js | 9 +-
.../apache/hadoop/hdfs/TestRollingUpgrade.java | 48 ++-
.../hadoop/hdfs/server/namenode/TestFsck.java | 159 ++++++++-
.../namenode/TestStoragePolicySummary.java | 201 +++++++++++
hadoop-mapreduce-project/CHANGES.txt | 6 +
.../java/org/apache/hadoop/mapred/JobConf.java | 6 +-
.../hadoop/mapred/TaskCompletionEvent.java | 32 +-
.../hadoop/mapreduce/TaskCompletionEvent.java | 32 +-
.../org/apache/hadoop/conf/TestJobConf.java | 23 +-
.../fs/azure/AzureNativeFileSystemStore.java | 29 +-
.../hadoop-sls/src/main/bin/rumen2sls.sh | 2 +-
hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 2 +-
hadoop-yarn-project/CHANGES.txt | 7 +
.../security/TestClientToAMTokens.java | 3 -
.../hadoop-yarn-server-web-proxy/pom.xml | 4 -
.../server/webproxy/WebAppProxyServlet.java | 46 +--
58 files changed, 2290 insertions(+), 880 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7788f977/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
[14/17] hadoop git commit: HDFS-7774. Unresolved symbols error while
compiling HDFS on Windows 7/32 bit. Contributed by Kiran Kumar M R.
Posted by zj...@apache.org.
HDFS-7774. Unresolved symbols error while compiling HDFS on Windows 7/32 bit. Contributed by Kiran Kumar M R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c6d5b37a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c6d5b37a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c6d5b37a
Branch: refs/heads/YARN-2928
Commit: c6d5b37a1c4c3acc190f5f4e27109594efb7be8c
Parents: 2214dab
Author: cnauroth <cn...@apache.org>
Authored: Thu Feb 26 13:37:46 2015 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Thu Feb 26 13:37:46 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 5 ++++-
.../hadoop-hdfs/src/main/native/libhdfs/os/windows/thread.c | 2 +-
.../main/native/libhdfs/os/windows/thread_local_storage.c | 8 ++++++++
4 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6d5b37a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 54b4057..ae83898 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1040,6 +1040,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7843. A truncated file is corrupted after rollback from a rolling
upgrade. (szetszwo)
+ HDFS-7774. Unresolved symbols error while compiling HDFS on Windows 7/32 bit.
+ (Kiran Kumar M R via cnauroth)
+
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6d5b37a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 5efce5c..2d402a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -440,10 +440,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
</goals>
<configuration>
<target>
+ <condition property="generator" value="Visual Studio 10" else="Visual Studio 10 Win64">
+ <equals arg1="Win32" arg2="${env.PLATFORM}" />
+ </condition>
<mkdir dir="${project.build.directory}/native"/>
<exec executable="cmake" dir="${project.build.directory}/native"
failonerror="true">
- <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_LIBWEBHDFS=${require.libwebhdfs} -DREQUIRE_FUSE=${require.fuse} -G 'Visual Studio 10 Win64'"/>
+ <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_LIBWEBHDFS=${require.libwebhdfs} -DREQUIRE_FUSE=${require.fuse} -G '${generator}'"/>
</exec>
<exec executable="msbuild" dir="${project.build.directory}/native"
failonerror="true">
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6d5b37a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/os/windows/thread.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/os/windows/thread.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/os/windows/thread.c
index 90450d8..f5cc2a7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/os/windows/thread.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/os/windows/thread.c
@@ -28,7 +28,7 @@
* @param toRun thread to run
* @return DWORD result of running thread (always 0)
*/
-static DWORD runThread(LPVOID toRun) {
+static DWORD WINAPI runThread(LPVOID toRun) {
const thread *t = toRun;
t->start(t->arg);
return 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6d5b37a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/os/windows/thread_local_storage.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/os/windows/thread_local_storage.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/os/windows/thread_local_storage.c
index 70ad152..4c415e1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/os/windows/thread_local_storage.c
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/os/windows/thread_local_storage.c
@@ -96,13 +96,21 @@ static void NTAPI tlsCallback(PVOID h, DWORD reason, PVOID pv)
* reference _tls_used, we guarantee that the binary retains the TLS directory.
* See Microsoft Visual Studio 10.0/VC/crt/src/tlssup.c .
*/
+#ifdef _WIN64
#pragma comment(linker, "/INCLUDE:_tls_used")
+#else
+#pragma comment(linker, "/INCLUDE:__tls_used")
+#endif
/*
* We must retain a pointer to the callback function. Force the linker to keep
* this symbol, even though it appears that nothing in our source code uses it.
*/
+#ifdef _WIN64
#pragma comment(linker, "/INCLUDE:pTlsCallback")
+#else
+#pragma comment(linker, "/INCLUDE:_pTlsCallback")
+#endif
/*
* Define constant pointer to our callback, and tell the linker to pin it into
[12/17] hadoop git commit: HDFS-7819. Log WARN message for the blocks
which are not in Block ID based layout (Rakesh R via Colin P. McCabe)
Posted by zj...@apache.org.
HDFS-7819. Log WARN message for the blocks which are not in Block ID based layout (Rakesh R via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f0c980ab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f0c980ab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f0c980ab
Branch: refs/heads/YARN-2928
Commit: f0c980abed3843923e0eb16b626fa27334195eda
Parents: dce8b9c
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Thu Feb 26 11:58:29 2015 -0800
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Thu Feb 26 11:58:29 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hdfs/server/datanode/DirectoryScanner.java | 26 +++++++++++++++++---
2 files changed, 25 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0c980ab/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e09714f..54b4057 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -679,6 +679,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7832. Show 'Last Modified' in Namenode's 'Browse Filesystem'
(vinayakumarb)
+ HDFS-7819. Log WARN message for the blocks which are not in Block ID based
+ layout (Rakesh R via Colin P. McCabe)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0c980ab/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index 71f976b..09c2914 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -597,14 +597,15 @@ public class DirectoryScanner implements Runnable {
for (String bpid : bpList) {
LinkedList<ScanInfo> report = new LinkedList<ScanInfo>();
File bpFinalizedDir = volume.getFinalizedDir(bpid);
- result.put(bpid, compileReport(volume, bpFinalizedDir, report));
+ result.put(bpid,
+ compileReport(volume, bpFinalizedDir, bpFinalizedDir, report));
}
return result;
}
/** Compile list {@link ScanInfo} for the blocks in the directory <dir> */
- private LinkedList<ScanInfo> compileReport(FsVolumeSpi vol, File dir,
- LinkedList<ScanInfo> report) {
+ private LinkedList<ScanInfo> compileReport(FsVolumeSpi vol,
+ File bpFinalizedDir, File dir, LinkedList<ScanInfo> report) {
File[] files;
try {
files = FileUtil.listFiles(dir);
@@ -622,12 +623,14 @@ public class DirectoryScanner implements Runnable {
*/
for (int i = 0; i < files.length; i++) {
if (files[i].isDirectory()) {
- compileReport(vol, files[i], report);
+ compileReport(vol, bpFinalizedDir, files[i], report);
continue;
}
if (!Block.isBlockFilename(files[i])) {
if (isBlockMetaFile("blk_", files[i].getName())) {
long blockId = Block.getBlockId(files[i].getName());
+ verifyFileLocation(files[i].getParentFile(), bpFinalizedDir,
+ blockId);
report.add(new ScanInfo(blockId, null, files[i], vol));
}
continue;
@@ -646,9 +649,24 @@ public class DirectoryScanner implements Runnable {
break;
}
}
+ verifyFileLocation(blockFile.getParentFile(), bpFinalizedDir,
+ blockId);
report.add(new ScanInfo(blockId, blockFile, metaFile, vol));
}
return report;
}
+
+ /**
+ * Verify whether the actual directory location of block file has the
+ * expected directory path computed using its block ID.
+ */
+ private void verifyFileLocation(File actualBlockDir,
+ File bpFinalizedDir, long blockId) {
+ File blockDir = DatanodeUtil.idToBlockDir(bpFinalizedDir, blockId);
+ if (actualBlockDir.compareTo(blockDir) != 0) {
+ LOG.warn("Block: " + blockId
+ + " has to be upgraded to block ID-based layout");
+ }
+ }
}
}
[16/17] hadoop git commit: MAPREDUCE-5612. Add javadoc for
TaskCompletionEvent.Status. Contributed by Chris Palmer.
Posted by zj...@apache.org.
MAPREDUCE-5612. Add javadoc for TaskCompletionEvent.Status. Contributed by Chris Palmer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bfbf076b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bfbf076b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bfbf076b
Branch: refs/heads/YARN-2928
Commit: bfbf076b7dcb71b59d85b05d0a6da88195ea9902
Parents: 1047c88
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Feb 26 15:05:17 2015 -0800
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Feb 26 15:05:17 2015 -0800
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 ++
.../hadoop/mapred/TaskCompletionEvent.java | 32 +++++++++++++++++++-
.../hadoop/mapreduce/TaskCompletionEvent.java | 32 +++++++++++++++++++-
3 files changed, 65 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfbf076b/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 6cb1754..f509d4e 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -308,6 +308,9 @@ Release 2.7.0 - UNRELEASED
MAPREDUCE-6264. Remove httpclient dependency from hadoop-mapreduce-client.
(Brahma Reddy Battula via aajisaka)
+ MAPREDUCE-5612. Add javadoc for TaskCompletionEvent.Status.
+ (Chris Palmer via aajisaka)
+
OPTIMIZATIONS
MAPREDUCE-6169. MergeQueue should release reference to the current item
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfbf076b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskCompletionEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskCompletionEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskCompletionEvent.java
index dc4d82e..2bb55a2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskCompletionEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskCompletionEvent.java
@@ -32,7 +32,37 @@ public class TaskCompletionEvent
extends org.apache.hadoop.mapreduce.TaskCompletionEvent {
@InterfaceAudience.Public
@InterfaceStability.Stable
- static public enum Status {FAILED, KILLED, SUCCEEDED, OBSOLETE, TIPFAILED};
+ /**
+ * Task Completion Statuses
+ */
+ static public enum Status {
+ /**
+ * Task Event Attempt failed but there are attempts remaining.
+ */
+ FAILED,
+ /**
+ * Task Event was killed.
+ */
+ KILLED,
+ /**
+ * Task Event was successful.
+ */
+ SUCCEEDED,
+ /**
+ * Used to Override a previously successful event status.
+ * Example: Map attempt runs and a SUCCEEDED event is sent. Later a task
+ * is retroactively failed due to excessive fetch failure during shuffle
+ * phase. When the retroactive attempt failure occurs, an OBSOLETE event is
+ * sent for the map attempt indicating the prior event is no longer valid.
+ */
+ OBSOLETE,
+ /**
+ * Task Event attempt failed and no further attempts exist.
+ * reached MAX attempts. When a reducer receives a TIPFAILED event it
+ * gives up trying to shuffle data from that map task.
+ */
+ TIPFAILED
+ }
public static final TaskCompletionEvent[] EMPTY_ARRAY =
new TaskCompletionEvent[0];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bfbf076b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCompletionEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCompletionEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCompletionEvent.java
index 31643a9..21c3823 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCompletionEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskCompletionEvent.java
@@ -36,7 +36,37 @@ import org.apache.hadoop.io.WritableUtils;
public class TaskCompletionEvent implements Writable{
@InterfaceAudience.Public
@InterfaceStability.Evolving
- static public enum Status {FAILED, KILLED, SUCCEEDED, OBSOLETE, TIPFAILED};
+ /**
+ * Task Completion Statuses
+ */
+ static public enum Status {
+ /**
+ * Task Event Attempt failed but there are attempts remaining.
+ */
+ FAILED,
+ /**
+ * Task Event was killed.
+ */
+ KILLED,
+ /**
+ * Task Event was successful.
+ */
+ SUCCEEDED,
+ /**
+ * Used to Override a previously successful event status.
+ * Example: Map attempt runs and a SUCCEEDED event is sent. Later a task
+ * is retroactively failed due to excessive fetch failure during shuffle
+ * phase. When the retroactive attempt failure occurs, an OBSOLETE event is
+ * sent for the map attempt indicating the prior event is no longer valid.
+ */
+ OBSOLETE,
+ /**
+ * Task Event attempt failed and no further attempts exist.
+ * reached MAX attempts. When a reducer receives a TIPFAILED event it
+ * gives up trying to shuffle data from that map task.
+ */
+ TIPFAILED
+ }
private int eventId;
private String taskTrackerHttp;
[06/17] hadoop git commit: HDFS-7537. Add "UNDER MIN REPL'D BLOCKS"
count to fsck. Contributed by GAO Rui
Posted by zj...@apache.org.
HDFS-7537. Add "UNDER MIN REPL'D BLOCKS" count to fsck. Contributed by GAO Rui
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/725cc499
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/725cc499
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/725cc499
Branch: refs/heads/YARN-2928
Commit: 725cc499f00abeeab9f58cbc778e65522eec9d98
Parents: 8c4f76a
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Thu Feb 26 11:45:56 2015 +0800
Committer: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Committed: Thu Feb 26 11:45:56 2015 +0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/namenode/NamenodeFsck.java | 36 +++++++--
.../hadoop/hdfs/server/namenode/TestFsck.java | 81 +++++++++++++++++++-
3 files changed, 111 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/725cc499/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 6dc7a0f..4523bf4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -673,6 +673,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7495. Remove updatePosition argument from DFSInputStream#getBlockAt()
(cmccabe)
+ HDFS-7537. Add "UNDER MIN REPL'D BLOCKS" count to fsck. (GAO Rui via
+ szetszwo)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/725cc499/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 5134f3c..36b4461 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -512,6 +512,9 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
res.totalReplicas += liveReplicas;
short targetFileReplication = file.getReplication();
res.numExpectedReplicas += targetFileReplication;
+ if(liveReplicas<minReplication){
+ res.numUnderMinReplicatedBlocks++;
+ }
if (liveReplicas > targetFileReplication) {
res.excessiveReplicas += (liveReplicas - targetFileReplication);
res.numOverReplicatedBlocks += 1;
@@ -858,6 +861,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
long corruptBlocks = 0L;
long excessiveReplicas = 0L;
long missingReplicas = 0L;
+ long numUnderMinReplicatedBlocks=0L;
long numOverReplicatedBlocks = 0L;
long numUnderReplicatedBlocks = 0L;
long numMisReplicatedBlocks = 0L; // blocks that do not satisfy block placement policy
@@ -874,10 +878,13 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
long totalReplicas = 0L;
final short replication;
+ final int minReplication;
Result(Configuration conf) {
this.replication = (short)conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY,
DFSConfigKeys.DFS_REPLICATION_DEFAULT);
+ this.minReplication = (short)conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY,
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_DEFAULT);
}
/**
@@ -925,15 +932,28 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
res.append(" (Total open file blocks (not validated): ").append(
totalOpenFilesBlocks).append(")");
}
- if (corruptFiles > 0) {
- res.append("\n ********************************").append(
- "\n CORRUPT FILES:\t").append(corruptFiles);
- if (missingSize > 0) {
- res.append("\n MISSING BLOCKS:\t").append(missingIds.size()).append(
- "\n MISSING SIZE:\t\t").append(missingSize).append(" B");
+ if (corruptFiles > 0 || numUnderMinReplicatedBlocks>0) {
+ res.append("\n ********************************");
+ if(numUnderMinReplicatedBlocks>0){
+ res.append("\n UNDER MIN REPL'D BLOCKS:\t").append(numUnderMinReplicatedBlocks);
+ if(totalBlocks>0){
+ res.append(" (").append(
+ ((float) (numUnderMinReplicatedBlocks * 100) / (float) totalBlocks))
+ .append(" %)");
+ }
+ res.append("\n ").append("DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY:\t")
+ .append(minReplication);
}
- if (corruptBlocks > 0) {
- res.append("\n CORRUPT BLOCKS: \t").append(corruptBlocks);
+ if(corruptFiles>0) {
+ res.append(
+ "\n CORRUPT FILES:\t").append(corruptFiles);
+ if (missingSize > 0) {
+ res.append("\n MISSING BLOCKS:\t").append(missingIds.size()).append(
+ "\n MISSING SIZE:\t\t").append(missingSize).append(" B");
+ }
+ if (corruptBlocks > 0) {
+ res.append("\n CORRUPT BLOCKS: \t").append(corruptBlocks);
+ }
}
res.append("\n ********************************");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/725cc499/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index 35a5a03..1053b5f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -693,7 +693,86 @@ public class TestFsck {
if (cluster != null) {cluster.shutdown();}
}
}
-
+
+ @Test
+ public void testUnderMinReplicatedBlock() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000);
+ // Set short retry timeouts so this test runs faster
+ conf.setInt(DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE, 10);
+ // Set minReplication to 2
+ short minReplication=2;
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY,minReplication);
+ FileSystem fs = null;
+ DFSClient dfsClient = null;
+ LocatedBlocks blocks = null;
+ int replicaCount = 0;
+ Random random = new Random();
+ String outStr = null;
+ short factor = 1;
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
+ cluster.waitActive();
+ fs = cluster.getFileSystem();
+ Path file1 = new Path("/testUnderMinReplicatedBlock");
+ DFSTestUtil.createFile(fs, file1, 1024, minReplication, 0);
+ // Wait until file replication has completed
+ DFSTestUtil.waitReplication(fs, file1, minReplication);
+ ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file1);
+
+ // Make sure filesystem is in healthy state
+ outStr = runFsck(conf, 0, true, "/");
+ System.out.println(outStr);
+ assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
+
+ // corrupt the first replica
+ File blockFile = cluster.getBlockFile(0, block);
+ if (blockFile != null && blockFile.exists()) {
+ RandomAccessFile raFile = new RandomAccessFile(blockFile, "rw");
+ FileChannel channel = raFile.getChannel();
+ String badString = "BADBAD";
+ int rand = random.nextInt((int) channel.size()/2);
+ raFile.seek(rand);
+ raFile.write(badString.getBytes());
+ raFile.close();
+ }
+
+ dfsClient = new DFSClient(new InetSocketAddress("localhost",
+ cluster.getNameNodePort()), conf);
+ blocks = dfsClient.getNamenode().
+ getBlockLocations(file1.toString(), 0, Long.MAX_VALUE);
+ replicaCount = blocks.get(0).getLocations().length;
+ while (replicaCount != factor) {
+ try {
+ Thread.sleep(100);
+ // Read the file to trigger reportBadBlocks
+ try {
+ IOUtils.copyBytes(fs.open(file1), new IOUtils.NullOutputStream(), conf,
+ true);
+ } catch (IOException ie) {
+ // Ignore exception
+ }
+ System.out.println("sleep in try: replicaCount="+replicaCount+" factor="+factor);
+ } catch (InterruptedException ignore) {
+ }
+ blocks = dfsClient.getNamenode().
+ getBlockLocations(file1.toString(), 0, Long.MAX_VALUE);
+ replicaCount = blocks.get(0).getLocations().length;
+ }
+
+ // Check if fsck reports the same
+ outStr = runFsck(conf, 0, true, "/");
+ System.out.println(outStr);
+ assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
+ assertTrue(outStr.contains("UNDER MIN REPL'D BLOCKS:\t1 (100.0 %)"));
+ assertTrue(outStr.contains("DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY:\t2"));
+ } finally {
+ if (cluster != null) {cluster.shutdown();}
+ }
+ }
+
+
/** Test if fsck can return -1 in case of failure
*
* @throws Exception
[10/17] hadoop git commit: YARN-3217. Remove httpclient dependency
from hadoop-yarn-server-web-proxy. Contributed by Brahma Reddy Battula.
Posted by zj...@apache.org.
YARN-3217. Remove httpclient dependency from hadoop-yarn-server-web-proxy. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/773b6515
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/773b6515
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/773b6515
Branch: refs/heads/YARN-2928
Commit: 773b6515ac51af3484824bd6f57685a9726a1e70
Parents: 0d4296f
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Fri Feb 27 00:22:46 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Fri Feb 27 00:24:29 2015 +0900
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop-yarn-server-web-proxy/pom.xml | 4 --
.../server/webproxy/WebAppProxyServlet.java | 46 ++++++++++----------
3 files changed, 26 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773b6515/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ac3cbb2..a635592 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -327,6 +327,9 @@ Release 2.7.0 - UNRELEASED
YARN-2797. Add -help to yarn logs and nodes CLI command.
(Jagadesh Kiran N via devaraj)
+ YARN-3217. Remove httpclient dependency from hadoop-yarn-server-web-proxy.
+ (Brahma Reddy Battula via ozawa).
+
OPTIMIZATIONS
YARN-2990. FairScheduler's delay-scheduling always waits for node-local and
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773b6515/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
index fdba1fe..9801064 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
@@ -79,10 +79,6 @@
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- </dependency>
- <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773b6515/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
index 47f7769..fd98c80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
@@ -40,13 +40,6 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.UriBuilder;
-import org.apache.commons.httpclient.Header;
-import org.apache.commons.httpclient.HostConfiguration;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.cookie.CookiePolicy;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.params.HttpClientParams;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
@@ -59,8 +52,15 @@ import org.apache.hadoop.yarn.util.TrackingUriPlugin;
import org.apache.hadoop.yarn.webapp.MimeType;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.http.Header;
+import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.params.ClientPNames;
+import org.apache.http.client.params.CookiePolicy;
import org.apache.http.client.utils.URLEncodedUtils;
+import org.apache.http.conn.params.ConnRoutePNames;
+import org.apache.http.impl.client.DefaultHttpClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -155,23 +155,22 @@ public class WebAppProxyServlet extends HttpServlet {
private static void proxyLink(HttpServletRequest req,
HttpServletResponse resp, URI link, Cookie c, String proxyHost)
throws IOException {
- org.apache.commons.httpclient.URI uri =
- new org.apache.commons.httpclient.URI(link.toString(), false);
- HttpClientParams params = new HttpClientParams();
- params.setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
- params.setBooleanParameter(HttpClientParams.ALLOW_CIRCULAR_REDIRECTS, true);
- HttpClient client = new HttpClient(params);
+ DefaultHttpClient client = new DefaultHttpClient();
+ client
+ .getParams()
+ .setParameter(ClientPNames.COOKIE_POLICY,
+ CookiePolicy.BROWSER_COMPATIBILITY)
+ .setBooleanParameter(ClientPNames.ALLOW_CIRCULAR_REDIRECTS, true);
// Make sure we send the request from the proxy address in the config
// since that is what the AM filter checks against. IP aliasing or
// similar could cause issues otherwise.
- HostConfiguration config = new HostConfiguration();
InetAddress localAddress = InetAddress.getByName(proxyHost);
if (LOG.isDebugEnabled()) {
LOG.debug("local InetAddress for proxy host: {}", localAddress);
}
- config.setLocalAddress(localAddress);
- HttpMethod method = new GetMethod(uri.getEscapedURI());
- method.setRequestHeader("Connection","close");
+ client.getParams()
+ .setParameter(ConnRoutePNames.LOCAL_ADDRESS, localAddress);
+ HttpGet httpGet = new HttpGet(link);
@SuppressWarnings("unchecked")
Enumeration<String> names = req.getHeaderNames();
while(names.hasMoreElements()) {
@@ -181,30 +180,31 @@ public class WebAppProxyServlet extends HttpServlet {
if (LOG.isDebugEnabled()) {
LOG.debug("REQ HEADER: {} : {}", name, value);
}
- method.setRequestHeader(name, value);
+ httpGet.setHeader(name, value);
}
}
String user = req.getRemoteUser();
if (user != null && !user.isEmpty()) {
- method.setRequestHeader("Cookie",
+ httpGet.setHeader("Cookie",
PROXY_USER_COOKIE_NAME + "=" + URLEncoder.encode(user, "ASCII"));
}
OutputStream out = resp.getOutputStream();
try {
- resp.setStatus(client.executeMethod(config, method));
- for(Header header : method.getResponseHeaders()) {
+ HttpResponse httpResp = client.execute(httpGet);
+ resp.setStatus(httpResp.getStatusLine().getStatusCode());
+ for (Header header : httpResp.getAllHeaders()) {
resp.setHeader(header.getName(), header.getValue());
}
if (c != null) {
resp.addCookie(c);
}
- InputStream in = method.getResponseBodyAsStream();
+ InputStream in = httpResp.getEntity().getContent();
if (in != null) {
IOUtils.copyBytes(in, out, 4096, true);
}
} finally {
- method.releaseConnection();
+ httpGet.releaseConnection();
}
}
[08/17] hadoop git commit: HDFS-7832. Show 'Last Modified' in
Namenode's 'Browse Filesystem' (Contributed by Vinayakumar B)
Posted by zj...@apache.org.
HDFS-7832. Show 'Last Modified' in Namenode's 'Browse Filesystem' (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/166eecf6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/166eecf6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/166eecf6
Branch: refs/heads/YARN-2928
Commit: 166eecf687765f369cd5bf210cc6d7bf493121ea
Parents: 71385f9
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Feb 26 14:36:09 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Feb 26 14:36:09 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop-hdfs/src/main/webapps/hdfs/explorer.html | 2 ++
.../hadoop-hdfs/src/main/webapps/hdfs/explorer.js | 9 ++++++++-
3 files changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/166eecf6/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4523bf4..e09714f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -676,6 +676,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7537. Add "UNDER MIN REPL'D BLOCKS" count to fsck. (GAO Rui via
szetszwo)
+ HDFS-7832. Show 'Last Modified' in Namenode's 'Browse Filesystem'
+ (vinayakumarb)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/166eecf6/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
index e1fdfa3..7b34044 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
@@ -112,6 +112,7 @@
<th>Owner</th>
<th>Group</th>
<th>Size</th>
+ <th>Last Modified</th>
<th>Replication</th>
<th>Block Size</th>
<th>Name</th>
@@ -124,6 +125,7 @@
<td>{owner}</td>
<td>{group}</td>
<td>{length|fmt_bytes}</td>
+ <td>{#helper_date_tostring value="{modificationTime}"/}</td>
<td>{replication}</td>
<td>{blockSize|fmt_bytes}</td>
<td><a style="cursor:pointer" inode-type="{type}" class="explorer-browse-links" inode-path="{pathSuffix}">{pathSuffix}</a></td>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/166eecf6/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
index 87d47fa..0a53dcd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
@@ -143,6 +143,12 @@
}
function browse_directory(dir) {
+ var HELPERS = {
+ 'helper_date_tostring' : function (chunk, ctx, bodies, params) {
+ var value = dust.helpers.tap(params.value, chunk, ctx);
+ return chunk.write('' + new Date(Number(value)).toLocaleString());
+ }
+ };
var url = '/webhdfs/v1' + dir + '?op=LISTSTATUS';
$.get(url, function(data) {
var d = get_response(data, "FileStatuses");
@@ -154,7 +160,8 @@
current_directory = dir;
$('#directory').val(dir);
window.location.hash = dir;
- dust.render('explorer', d, function(err, out) {
+ var base = dust.makeBase(HELPERS);
+ dust.render('explorer', base.push(d), function(err, out) {
$('#panel').html(out);
$('.explorer-browse-links').click(function() {
[04/17] hadoop git commit: HDFS-7843. A truncated file is corrupted
after rollback from a rolling upgrade.
Posted by zj...@apache.org.
HDFS-7843. A truncated file is corrupted after rollback from a rolling upgrade.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/606f5b51
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/606f5b51
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/606f5b51
Branch: refs/heads/YARN-2928
Commit: 606f5b517ffbeae0140a8c80b4cddc012c7fb3c4
Parents: d140d76
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Thu Feb 26 10:14:40 2015 +0800
Committer: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Committed: Thu Feb 26 10:14:40 2015 +0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hdfs/server/namenode/FSNamesystem.java | 3 ++
.../apache/hadoop/hdfs/TestRollingUpgrade.java | 48 ++++++++++++++------
3 files changed, 40 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/606f5b51/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e0f9267..f8b0c37 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1026,6 +1026,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7763. fix zkfc hung issue due to not catching exception in a corner
case. (Liang Xie via wang)
+ HDFS-7843. A truncated file is corrupted after rollback from a rolling
+ upgrade. (szetszwo)
+
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/606f5b51/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 120a597..bbab09e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2077,6 +2077,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if(!isUpgradeFinalized()) {
return true;
}
+ if (isRollingUpgrade()) {
+ return true;
+ }
return file.isBlockInLatestSnapshot(blk);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/606f5b51/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
index 8e7b4b1..9746049 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
@@ -23,9 +23,11 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
@@ -36,6 +38,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.namenode.FSImage;
import org.apache.hadoop.hdfs.server.namenode.NNStorage;
import org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode;
+import org.apache.hadoop.hdfs.server.namenode.TestFileTruncate;
import org.apache.hadoop.hdfs.tools.DFSAdmin;
import org.apache.hadoop.io.IOUtils;
import org.junit.Assert;
@@ -260,42 +263,50 @@ public class TestRollingUpgrade {
final Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = null;
try {
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
cluster.waitActive();
final Path foo = new Path("/foo");
final Path bar = new Path("/bar");
cluster.getFileSystem().mkdirs(foo);
- startRollingUpgrade(foo, bar, cluster);
+ final Path file = new Path(foo, "file");
+ final byte[] data = new byte[1024];
+ DFSUtil.getRandom().nextBytes(data);
+ final FSDataOutputStream out = cluster.getFileSystem().create(file);
+ out.write(data, 0, data.length);
+ out.close();
+
+ startRollingUpgrade(foo, bar, file, data, cluster);
cluster.getFileSystem().rollEdits();
cluster.getFileSystem().rollEdits();
- rollbackRollingUpgrade(foo, bar, cluster);
+ rollbackRollingUpgrade(foo, bar, file, data, cluster);
- startRollingUpgrade(foo, bar, cluster);
+ startRollingUpgrade(foo, bar, file, data, cluster);
cluster.getFileSystem().rollEdits();
cluster.getFileSystem().rollEdits();
- rollbackRollingUpgrade(foo, bar, cluster);
+ rollbackRollingUpgrade(foo, bar, file, data, cluster);
- startRollingUpgrade(foo, bar, cluster);
+ startRollingUpgrade(foo, bar, file, data, cluster);
cluster.restartNameNode();
- rollbackRollingUpgrade(foo, bar, cluster);
+ rollbackRollingUpgrade(foo, bar, file, data, cluster);
- startRollingUpgrade(foo, bar, cluster);
+ startRollingUpgrade(foo, bar, file, data, cluster);
cluster.restartNameNode();
- rollbackRollingUpgrade(foo, bar, cluster);
+ rollbackRollingUpgrade(foo, bar, file, data, cluster);
- startRollingUpgrade(foo, bar, cluster);
- rollbackRollingUpgrade(foo, bar, cluster);
+ startRollingUpgrade(foo, bar, file, data, cluster);
+ rollbackRollingUpgrade(foo, bar, file, data, cluster);
- startRollingUpgrade(foo, bar, cluster);
- rollbackRollingUpgrade(foo, bar, cluster);
+ startRollingUpgrade(foo, bar, file, data, cluster);
+ rollbackRollingUpgrade(foo, bar, file, data, cluster);
} finally {
if(cluster != null) cluster.shutdown();
}
}
private static void startRollingUpgrade(Path foo, Path bar,
+ Path file, byte[] data,
MiniDFSCluster cluster) throws IOException {
final DistributedFileSystem dfs = cluster.getFileSystem();
@@ -305,18 +316,27 @@ public class TestRollingUpgrade {
dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
dfs.mkdirs(bar);
-
Assert.assertTrue(dfs.exists(foo));
Assert.assertTrue(dfs.exists(bar));
+
+ //truncate a file
+ final int newLength = DFSUtil.getRandom().nextInt(data.length - 1) + 1;
+ dfs.truncate(file, newLength);
+ TestFileTruncate.checkBlockRecovery(file, dfs);
+ AppendTestUtil.checkFullFile(dfs, file, newLength, data);
}
private static void rollbackRollingUpgrade(Path foo, Path bar,
+ Path file, byte[] data,
MiniDFSCluster cluster) throws IOException {
+ final DataNodeProperties dnprop = cluster.stopDataNode(0);
cluster.restartNameNode("-rollingUpgrade", "rollback");
+ cluster.restartDataNode(dnprop, true);
final DistributedFileSystem dfs = cluster.getFileSystem();
Assert.assertTrue(dfs.exists(foo));
Assert.assertFalse(dfs.exists(bar));
+ AppendTestUtil.checkFullFile(dfs, file, data.length, data);
}
@Test
[15/17] hadoop git commit: MAPREDUCE-6223.
TestJobConf#testNegativeValueForTaskVmem failures. (Varun Saxena via kasha)
Posted by zj...@apache.org.
MAPREDUCE-6223. TestJobConf#testNegativeValueForTaskVmem failures. (Varun Saxena via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1047c883
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1047c883
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1047c883
Branch: refs/heads/YARN-2928
Commit: 1047c883ba01a252dbd4203e525fb1ff8ea313aa
Parents: c6d5b37
Author: Karthik Kambatla <ka...@apache.org>
Authored: Thu Feb 26 14:24:19 2015 -0800
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Thu Feb 26 14:24:19 2015 -0800
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/mapred/JobConf.java | 6 ++---
.../org/apache/hadoop/conf/TestJobConf.java | 23 ++++++++++----------
3 files changed, 17 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1047c883/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index d2c4ab8..6cb1754 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -390,6 +390,9 @@ Release 2.7.0 - UNRELEASED
MAPREDUCE-2815. JavaDoc does not generate correctly for
MultithreadedMapRunner. (Chris Palmer via aajisaka)
+ MAPREDUCE-6223. TestJobConf#testNegativeValueForTaskVmem failures.
+ (Varun Saxena via kasha)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1047c883/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
index 98a643f..315c829 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
@@ -1826,8 +1826,7 @@ public class JobConf extends Configuration {
public long getMemoryForMapTask() {
long value = getDeprecatedMemoryValue();
if (value < 0) {
- return getLong(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY,
- JobContext.DEFAULT_MAP_MEMORY_MB);
+ return getMemoryRequired(TaskType.MAP);
}
return value;
}
@@ -1853,8 +1852,7 @@ public class JobConf extends Configuration {
public long getMemoryForReduceTask() {
long value = getDeprecatedMemoryValue();
if (value < 0) {
- return getLong(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY,
- JobContext.DEFAULT_REDUCE_MEMORY_MB);
+ return getMemoryRequired(TaskType.REDUCE);
}
return value;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1047c883/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
index e380d92..f67ba1f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestJobConf.java
@@ -19,9 +19,7 @@ package org.apache.hadoop.conf;
import org.junit.Assert;
import org.junit.Test;
-
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.JobContext;
import org.apache.hadoop.mapreduce.MRJobConfig;
public class TestJobConf {
@@ -87,8 +85,10 @@ public class TestJobConf {
configuration.set("mapred.task.maxvmem" , String.valueOf(-1));
configuration.set(MRJobConfig.MAP_MEMORY_MB,"-1");
configuration.set(MRJobConfig.REDUCE_MEMORY_MB,"-1");
- Assert.assertEquals(configuration.getMemoryForMapTask(),-1);
- Assert.assertEquals(configuration.getMemoryForReduceTask(),-1);
+ Assert.assertEquals(configuration.getMemoryForMapTask(),
+ MRJobConfig.DEFAULT_MAP_MEMORY_MB);
+ Assert.assertEquals(configuration.getMemoryForReduceTask(),
+ MRJobConfig.DEFAULT_REDUCE_MEMORY_MB);
configuration = new JobConf();
configuration.set("mapred.task.maxvmem" , String.valueOf(2*1024 * 1024));
@@ -96,7 +96,6 @@ public class TestJobConf {
configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "3");
Assert.assertEquals(configuration.getMemoryForMapTask(),2);
Assert.assertEquals(configuration.getMemoryForReduceTask(),2);
-
}
/**
@@ -106,7 +105,7 @@ public class TestJobConf {
@Test
public void testNegativeValueForTaskVmem() {
JobConf configuration = new JobConf();
-
+
configuration.set(JobConf.MAPRED_TASK_MAXVMEM_PROPERTY, "-3");
Assert.assertEquals(MRJobConfig.DEFAULT_MAP_MEMORY_MB,
configuration.getMemoryForMapTask());
@@ -129,10 +128,12 @@ public class TestJobConf {
configuration.set(MRJobConfig.MAP_MEMORY_MB, "-5");
configuration.set(MRJobConfig.REDUCE_MEMORY_MB, "-6");
- Assert.assertEquals(-5, configuration.getMemoryForMapTask());
- Assert.assertEquals(-6, configuration.getMemoryForReduceTask());
+ Assert.assertEquals(MRJobConfig.DEFAULT_MAP_MEMORY_MB,
+ configuration.getMemoryForMapTask());
+ Assert.assertEquals(MRJobConfig.DEFAULT_REDUCE_MEMORY_MB,
+ configuration.getMemoryForReduceTask());
}
-
+
/**
* Test deprecated accessor and mutator method for mapred.task.maxvmem
*/
@@ -144,13 +145,13 @@ public class TestJobConf {
configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(300));
configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(-1));
Assert.assertEquals(
- configuration.getMaxVirtualMemoryForTask(), 300 * 1024 * 1024);
+ configuration.getMaxVirtualMemoryForTask(), 1024 * 1024 * 1024);
configuration = new JobConf();
configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(-1));
configuration.set(MRJobConfig.REDUCE_MEMORY_MB, String.valueOf(200));
Assert.assertEquals(
- configuration.getMaxVirtualMemoryForTask(), 200 * 1024 * 1024);
+ configuration.getMaxVirtualMemoryForTask(), 1024 * 1024 * 1024);
configuration = new JobConf();
configuration.set(MRJobConfig.MAP_MEMORY_MB, String.valueOf(-1));
[05/17] hadoop git commit: HDFS-7460. Rewrite httpfs to use new shell
framework (John Smith via aw)
Posted by zj...@apache.org.
HDFS-7460. Rewrite httpfs to use new shell framework (John Smith via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8c4f76aa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8c4f76aa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8c4f76aa
Branch: refs/heads/YARN-2928
Commit: 8c4f76aa20e75635bd6d3de14924ec246a8a071a
Parents: 606f5b5
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed Feb 25 18:57:41 2015 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Wed Feb 25 18:57:41 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 2 +-
.../src/main/conf/httpfs-env.sh | 51 +++--
.../src/main/libexec/httpfs-config.sh | 208 ++++---------------
.../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 116 +++++++----
.../src/main/tomcat/ssl-server.xml | 135 ------------
.../src/main/tomcat/ssl-server.xml.conf | 135 ++++++++++++
.../src/site/apt/ServerSetup.apt.vm | 159 --------------
.../src/site/apt/UsingHttpTools.apt.vm | 87 --------
.../src/site/apt/index.apt.vm | 83 --------
.../src/site/markdown/ServerSetup.md.vm | 121 +++++++++++
.../src/site/markdown/UsingHttpTools.md | 62 ++++++
.../src/site/markdown/index.md | 52 +++++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
13 files changed, 533 insertions(+), 680 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c4f76aa/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index 4c42ef9..ddc6033 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -524,7 +524,7 @@
<copy file="${basedir}/src/main/tomcat/server.xml"
toDir="${httpfs.tomcat.dist.dir}/conf"/>
<delete file="${httpfs.tomcat.dist.dir}/conf/ssl-server.xml"/>
- <copy file="${basedir}/src/main/tomcat/ssl-server.xml"
+ <copy file="${basedir}/src/main/tomcat/ssl-server.xml.conf"
toDir="${httpfs.tomcat.dist.dir}/conf"/>
<delete file="${httpfs.tomcat.dist.dir}/conf/logging.properties"/>
<copy file="${basedir}/src/main/tomcat/logging.properties"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c4f76aa/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
index a2701d4..0e8cc40 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
@@ -14,40 +14,59 @@
#
# Set httpfs specific environment variables here.
-
-# Settings for the Embedded Tomcat that runs HttpFS
-# Java System properties for HttpFS should be specified in this variable
#
-# export CATALINA_OPTS=
-
-# HttpFS logs directory
+# hadoop-env.sh is read prior to this file.
#
-# export HTTPFS_LOG=${HTTPFS_HOME}/logs
-# HttpFS temporary directory
+# HTTPFS temporary directory
#
-# export HTTPFS_TEMP=${HTTPFS_HOME}/temp
+# export HTTPFS_TEMP=${HADOOP_PREFIX}/temp
-# The HTTP port used by HttpFS
+# The HTTP port used by HTTPFS
#
# export HTTPFS_HTTP_PORT=14000
-# The Admin port used by HttpFS
+# The Admin port used by HTTPFS
#
-# export HTTPFS_ADMIN_PORT=`expr ${HTTPFS_HTTP_PORT} + 1`
+# export HTTPFS_ADMIN_PORT=$((HTTPFS_HTTP_PORT + 1))
-# The hostname HttpFS server runs on
+# The maximum number of Tomcat handler threads
#
-# export HTTPFS_HTTP_HOSTNAME=`hostname -f`
+# export HTTPFS_MAX_THREADS=1000
-# Indicates if HttpFS is using SSL
+# The hostname HttpFS server runs on
#
-# export HTTPFS_SSL_ENABLED=false
+# export HTTPFS_HTTP_HOSTNAME=$(hostname -f)
# The location of the SSL keystore if using SSL
#
# export HTTPFS_SSL_KEYSTORE_FILE=${HOME}/.keystore
+#
# The password of the SSL keystore if using SSL
#
# export HTTPFS_SSL_KEYSTORE_PASS=password
+
+##
+## Tomcat specific settings
+##
+#
+# Location of tomcat
+#
+# export HTTPFS_CATALINA_HOME=${HADOOP_PREFIX}/share/hadoop/httpfs/tomcat
+
+# Java System properties for HTTPFS should be specified in this variable.
+# The java.library.path and hadoop.home.dir properties are automatically
+# configured. In order to supplement java.library.path,
+# one should add to the JAVA_LIBRARY_PATH env var.
+#
+# export CATALINA_OPTS=
+
+# PID file
+#
+# export CATALINA_PID=${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-httpfs.pid
+
+# Output file
+#
+# export CATALINA_OUT=${HTTPFS_LOG}/hadoop-${HADOOP_IDENT_STRING}-httpfs-${HOSTNAME}.out
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c4f76aa/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
index d4db28f..e520e08 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -13,183 +13,63 @@
# limitations under the License.
#
-# resolve links - $0 may be a softlink
-PRG="${0}"
-
-while [ -h "${PRG}" ]; do
- ls=`ls -ld "${PRG}"`
- link=`expr "$ls" : '.*-> \(.*\)$'`
- if expr "$link" : '/.*' > /dev/null; then
- PRG="$link"
- else
- PRG=`dirname "${PRG}"`/"$link"
+function hadoop_subproject_init
+{
+ local this
+ local binparent
+ local varlist
+
+ if [[ -z "${HADOOP_HTTPFS_ENV_PROCESSED}" ]]; then
+ if [[ -e "${HADOOP_CONF_DIR}/httpfs-env.sh" ]]; then
+ . "${HADOOP_CONF_DIR}/httpfs-env.sh"
+ export HADOOP_HTTPFS_ENV_PROCESSED=true
+ fi
fi
-done
-BASEDIR=`dirname ${PRG}`
-BASEDIR=`cd ${BASEDIR}/..;pwd`
+ export HADOOP_CATALINA_PREFIX=httpfs
+ export HADOOP_CATALINA_TEMP="${HTTPFS_TEMP:-${HADOOP_PREFIX}/temp}"
-function print() {
- if [ "${HTTPFS_SILENT}" != "true" ]; then
- echo "$@"
- fi
-}
+ hadoop_deprecate_envvar HTTPFS_CONFIG HADOOP_CONF_DIR
-# if HTTPFS_HOME is already set warn it will be ignored
-#
-if [ "${HTTPFS_HOME}" != "" ]; then
- echo "WARNING: current setting of HTTPFS_HOME ignored"
-fi
+ hadoop_deprecate_envvar HTTPFS_LOG HADOOP_LOG_DIR
-print
+ export HADOOP_CATALINA_CONFIG="${HADOOP_CONF_DIR}"
+ export HADOOP_CATALINA_LOG="${HADOOP_LOG_DIR}"
-# setting HTTPFS_HOME to the installation dir, it cannot be changed
-#
-export HTTPFS_HOME=${BASEDIR}
-httpfs_home=${HTTPFS_HOME}
-print "Setting HTTPFS_HOME: ${HTTPFS_HOME}"
+ export HTTPFS_HTTP_HOSTNAME=${HTTPFS_HTTP_HOSTNAME:-$(hostname -f)}
-# if the installation has a env file, source it
-# this is for native packages installations
-#
-if [ -e "${HTTPFS_HOME}/bin/httpfs-env.sh" ]; then
- print "Sourcing: ${HTTPFS_HOME}/bin/httpfs-env.sh"
- source ${HTTPFS_HOME}/bin/httpfs-env.sh
- grep "^ *export " ${HTTPFS_HOME}/bin/httpfs-env.sh | sed 's/ *export/ setting/'
-fi
+ export HADOOP_CATALINA_HTTP_PORT="${HTTPFS_HTTP_PORT:-14000}"
+ export HADOOP_CATALINA_ADMIN_PORT="${HTTPFS_ADMIN_PORT:-$((HADOOP_CATALINA_HTTP_PORT+1))}"
+ export HADOOP_CATALINA_MAX_THREADS="${HTTPFS_MAX_THREADS:-150}"
-# verify that the sourced env file didn't change HTTPFS_HOME
-# if so, warn and revert
-#
-if [ "${HTTPFS_HOME}" != "${httpfs_home}" ]; then
- print "WARN: HTTPFS_HOME resetting to ''${HTTPFS_HOME}'' ignored"
- export HTTPFS_HOME=${httpfs_home}
- print " using HTTPFS_HOME: ${HTTPFS_HOME}"
-fi
+ export HTTPFS_SSL_ENABLED=${HTTPFS_SSL_ENABLED:-false}
-if [ "${HTTPFS_CONFIG}" = "" ]; then
- export HTTPFS_CONFIG=${HTTPFS_HOME}/etc/hadoop
- print "Setting HTTPFS_CONFIG: ${HTTPFS_CONFIG}"
-else
- print "Using HTTPFS_CONFIG: ${HTTPFS_CONFIG}"
-fi
-httpfs_config=${HTTPFS_CONFIG}
+ export HADOOP_CATALINA_SSL_KEYSTORE_FILE="${HTTPFS_SSL_KEYSTORE_FILE:-${HOME}/.keystore}"
-# if the configuration dir has a env file, source it
-#
-if [ -e "${HTTPFS_CONFIG}/httpfs-env.sh" ]; then
- print "Sourcing: ${HTTPFS_CONFIG}/httpfs-env.sh"
- source ${HTTPFS_CONFIG}/httpfs-env.sh
- grep "^ *export " ${HTTPFS_CONFIG}/httpfs-env.sh | sed 's/ *export/ setting/'
-fi
+ export CATALINA_BASE="${CATALINA_BASE:-${HADOOP_PREFIX}/share/hadoop/httpfs/tomcat}"
+ export HADOOP_CATALINA_HOME="${HTTPFS_CATALINA_HOME:-${CATALINA_BASE}}"
-# verify that the sourced env file didn't change HTTPFS_HOME
-# if so, warn and revert
-#
-if [ "${HTTPFS_HOME}" != "${httpfs_home}" ]; then
- echo "WARN: HTTPFS_HOME resetting to ''${HTTPFS_HOME}'' ignored"
- export HTTPFS_HOME=${httpfs_home}
-fi
-
-# verify that the sourced env file didn't change HTTPFS_CONFIG
-# if so, warn and revert
-#
-if [ "${HTTPFS_CONFIG}" != "${httpfs_config}" ]; then
- echo "WARN: HTTPFS_CONFIG resetting to ''${HTTPFS_CONFIG}'' ignored"
- export HTTPFS_CONFIG=${httpfs_config}
-fi
-
-if [ "${HTTPFS_LOG}" = "" ]; then
- export HTTPFS_LOG=${HTTPFS_HOME}/logs
- print "Setting HTTPFS_LOG: ${HTTPFS_LOG}"
-else
- print "Using HTTPFS_LOG: ${HTTPFS_LOG}"
-fi
+ export CATALINA_OUT="${CATALINA_OUT:-${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-httpfs-${HOSTNAME}.out}"
-if [ ! -f ${HTTPFS_LOG} ]; then
- mkdir -p ${HTTPFS_LOG}
-fi
-
-if [ "${HTTPFS_TEMP}" = "" ]; then
- export HTTPFS_TEMP=${HTTPFS_HOME}/temp
- print "Setting HTTPFS_TEMP: ${HTTPFS_TEMP}"
-else
- print "Using HTTPFS_TEMP: ${HTTPFS_TEMP}"
-fi
+ export CATALINA_PID="${CATALINA_PID:-${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-httpfs.pid}"
-if [ ! -f ${HTTPFS_TEMP} ]; then
- mkdir -p ${HTTPFS_TEMP}
-fi
-
-if [ "${HTTPFS_HTTP_PORT}" = "" ]; then
- export HTTPFS_HTTP_PORT=14000
- print "Setting HTTPFS_HTTP_PORT: ${HTTPFS_HTTP_PORT}"
-else
- print "Using HTTPFS_HTTP_PORT: ${HTTPFS_HTTP_PORT}"
-fi
-
-if [ "${HTTPFS_ADMIN_PORT}" = "" ]; then
- export HTTPFS_ADMIN_PORT=`expr $HTTPFS_HTTP_PORT + 1`
- print "Setting HTTPFS_ADMIN_PORT: ${HTTPFS_ADMIN_PORT}"
-else
- print "Using HTTPFS_ADMIN_PORT: ${HTTPFS_ADMIN_PORT}"
-fi
-
-if [ "${HTTPFS_HTTP_HOSTNAME}" = "" ]; then
- export HTTPFS_HTTP_HOSTNAME=`hostname -f`
- print "Setting HTTPFS_HTTP_HOSTNAME: ${HTTPFS_HTTP_HOSTNAME}"
-else
- print "Using HTTPFS_HTTP_HOSTNAME: ${HTTPFS_HTTP_HOSTNAME}"
-fi
-
-if [ "${HTTPFS_SSL_ENABLED}" = "" ]; then
- export HTTPFS_SSL_ENABLED="false"
- print "Setting HTTPFS_SSL_ENABLED: ${HTTPFS_SSL_ENABLED}"
-else
- print "Using HTTPFS_SSL_ENABLED: ${HTTPFS_SSL_ENABLED}"
-fi
-
-if [ "${HTTPFS_SSL_KEYSTORE_FILE}" = "" ]; then
- export HTTPFS_SSL_KEYSTORE_FILE=${HOME}/.keystore
- print "Setting HTTPFS_SSL_KEYSTORE_FILE: ${HTTPFS_SSL_KEYSTORE_FILE}"
-else
- print "Using HTTPFS_SSL_KEYSTORE_FILE: ${HTTPFS_SSL_KEYSTORE_FILE}"
-fi
-
-if [ "${HTTPFS_SSL_KEYSTORE_PASS}" = "" ]; then
- export HTTPFS_SSL_KEYSTORE_PASS=password
- print "Setting HTTPFS_SSL_KEYSTORE_PASS: ${HTTPFS_SSL_KEYSTORE_PASS}"
-else
- print "Using HTTPFS_SSL_KEYSTORE_PASS: ${HTTPFS_SSL_KEYSTORE_PASS}"
-fi
-
-if [ "${CATALINA_BASE}" = "" ]; then
- export CATALINA_BASE=${HTTPFS_HOME}/share/hadoop/httpfs/tomcat
- print "Setting CATALINA_BASE: ${CATALINA_BASE}"
-else
- print "Using CATALINA_BASE: ${CATALINA_BASE}"
-fi
-
-if [ "${HTTPFS_CATALINA_HOME}" = "" ]; then
- export HTTPFS_CATALINA_HOME=${CATALINA_BASE}
- print "Setting HTTPFS_CATALINA_HOME: ${HTTPFS_CATALINA_HOME}"
-else
- print "Using HTTPFS_CATALINA_HOME: ${HTTPFS_CATALINA_HOME}"
-fi
-
-if [ "${CATALINA_OUT}" = "" ]; then
- export CATALINA_OUT=${HTTPFS_LOG}/httpfs-catalina.out
- print "Setting CATALINA_OUT: ${CATALINA_OUT}"
-else
- print "Using CATALINA_OUT: ${CATALINA_OUT}"
-fi
+ if [[ -n "${HADOOP_SHELL_SCRIPT_DEBUG}" ]]; then
+ varlist=$(env | egrep '(^HTTPFS|^CATALINA)' | cut -f1 -d= | grep -v _PASS)
+ for i in ${varlist}; do
+ hadoop_debug "Setting ${i} to ${!i}"
+ done
+ fi
+}
-if [ "${CATALINA_PID}" = "" ]; then
- export CATALINA_PID=/tmp/httpfs.pid
- print "Setting CATALINA_PID: ${CATALINA_PID}"
+if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
+ [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]]; then
+ . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh"
+elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
+ . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
+elif [[ -e "${HADOOP_PREFIX}/libexec/hadoop-config.sh" ]]; then
+ . "${HADOOP_PREFIX}/libexec/hadoop-config.sh"
else
- print "Using CATALINA_PID: ${CATALINA_PID}"
+ echo "ERROR: Hadoop common not found." 2>&1
+ exit 1
fi
-
-print
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c4f76aa/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
index 65903dc..72dca93 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -13,53 +13,99 @@
# limitations under the License.
#
-# resolve links - $0 may be a softlink
-PRG="${0}"
+function hadoop_usage()
+{
+ echo "Usage: httpfs.sh [--config confdir] [--debug] --daemon start|status|stop"
+ echo " httpfs.sh [--config confdir] [--debug] COMMAND"
+ echo " where COMMAND is one of:"
+ echo " run Start httpfs in the current window"
+ echo " run -security Start in the current window with security manager"
+ echo " start Start httpfs in a separate window"
+ echo " start -security Start in a separate window with security manager"
+ echo " status Return the LSB compliant status"
+ echo " stop Stop httpfs, waiting up to 5 seconds for the process to end"
+ echo " stop n Stop httpfs, waiting up to n seconds for the process to end"
+ echo " stop -force Stop httpfs, wait up to 5 seconds and then use kill -KILL if still running"
+ echo " stop n -force Stop httpfs, wait up to n seconds and then use kill -KILL if still running"
+}
-while [ -h "${PRG}" ]; do
- ls=`ls -ld "${PRG}"`
- link=`expr "$ls" : '.*-> \(.*\)$'`
- if expr "$link" : '/.*' > /dev/null; then
- PRG="$link"
- else
- PRG=`dirname "${PRG}"`/"$link"
- fi
-done
-
-BASEDIR=`dirname ${PRG}`
-BASEDIR=`cd ${BASEDIR}/..;pwd`
+# let's locate libexec...
+if [[ -n "${HADOOP_PREFIX}" ]]; then
+ DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec"
+else
+ this="${BASH_SOURCE-$0}"
+ bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+ DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
+fi
-source ${HADOOP_LIBEXEC_DIR:-${BASEDIR}/libexec}/httpfs-config.sh
+HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}"
+# shellcheck disable=SC2034
+HADOOP_NEW_CONFIG=true
+if [[ -f "${HADOOP_LIBEXEC_DIR}/httpfs-config.sh" ]]; then
+ . "${HADOOP_LIBEXEC_DIR}/httpfs-config.sh"
+else
+ echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/httpfs-config.sh." 2>&1
+ exit 1
+fi
-# The Java System property 'httpfs.http.port' it is not used by HttpFS,
+# The Java System property 'httpfs.http.port' it is not used by Kms,
# it is used in Tomcat's server.xml configuration file
#
-print "Using CATALINA_OPTS: ${CATALINA_OPTS}"
-catalina_opts="-Dhttpfs.home.dir=${HTTPFS_HOME}";
-catalina_opts="${catalina_opts} -Dhttpfs.config.dir=${HTTPFS_CONFIG}";
-catalina_opts="${catalina_opts} -Dhttpfs.log.dir=${HTTPFS_LOG}";
-catalina_opts="${catalina_opts} -Dhttpfs.temp.dir=${HTTPFS_TEMP}";
-catalina_opts="${catalina_opts} -Dhttpfs.admin.port=${HTTPFS_ADMIN_PORT}";
-catalina_opts="${catalina_opts} -Dhttpfs.http.port=${HTTPFS_HTTP_PORT}";
-catalina_opts="${catalina_opts} -Dhttpfs.http.hostname=${HTTPFS_HTTP_HOSTNAME}";
-catalina_opts="${catalina_opts} -Dhttpfs.ssl.enabled=${HTTPFS_SSL_ENABLED}";
-catalina_opts="${catalina_opts} -Dhttpfs.ssl.keystore.file=${HTTPFS_SSL_KEYSTORE_FILE}";
-catalina_opts="${catalina_opts} -Dhttpfs.ssl.keystore.pass=${HTTPFS_SSL_KEYSTORE_PASS}";
+# Mask the trustStorePassword
+# shellcheck disable=SC2086
+CATALINA_OPTS_DISP="$(echo ${CATALINA_OPTS} | sed -e 's/trustStorePassword=[^ ]*/trustStorePassword=***/')"
-print "Adding to CATALINA_OPTS: ${catalina_opts}"
+hadoop_debug "Using CATALINA_OPTS: ${CATALINA_OPTS_DISP}"
-export CATALINA_OPTS="${CATALINA_OPTS} ${catalina_opts}"
+# We're using hadoop-common, so set up some stuff it might need:
+hadoop_finalize
+
+hadoop_verify_logdir
+
+if [[ $# = 0 ]]; then
+ case "${HADOOP_DAEMON_MODE}" in
+ status)
+ hadoop_status_daemon "${CATALINA_PID}"
+ exit
+ ;;
+ start)
+ set -- "start"
+ ;;
+ stop)
+ set -- "stop"
+ ;;
+ esac
+fi
+
+hadoop_finalize_catalina_opts
+export CATALINA_OPTS
# A bug in catalina.sh script does not use CATALINA_OPTS for stopping the server
#
-if [ "${1}" = "stop" ]; then
+if [[ "${1}" = "stop" ]]; then
export JAVA_OPTS=${CATALINA_OPTS}
fi
-if [ "${HTTPFS_SILENT}" != "true" ]; then
- exec ${HTTPFS_CATALINA_HOME}/bin/catalina.sh "$@"
-else
- exec ${HTTPFS_CATALINA_HOME}/bin/catalina.sh "$@" > /dev/null
+# If ssl, the populate the passwords into ssl-server.xml before starting tomcat
+#
+# HTTPFS_SSL_KEYSTORE_PASS is a bit odd.
+# if undefined, then the if test will not enable ssl on its own
+# if "", set it to "password".
+# if custom, use provided password
+#
+if [[ -f "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml.conf" ]]; then
+ if [[ -n "${HTTPFS_SSL_KEYSTORE_PASS+x}" ]] || [[ -n "${HTTPFS_SSL_TRUSTSTORE_PASS}" ]]; then
+ export HTTPFS_SSL_KEYSTORE_PASS=${HTTPFS_SSL_KEYSTORE_PASS:-password}
+ sed -e 's/_httpfs_ssl_keystore_pass_/'${HTTPFS_SSL_KEYSTORE_PASS}'/g' \
+ -e 's/_httpfs_ssl_truststore_pass_/'${HTTPFS_SSL_TRUSTSTORE_PASS}'/g' \
+ "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml.conf" \
+ > "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml"
+ chmod 700 "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml" >/dev/null 2>&1
+ fi
fi
+hadoop_add_param CATALINA_OPTS -Dhttpfs.http.hostname "-Dhttpfs.http.hostname=${HTTPFS_HOST_NAME}"
+hadoop_add_param CATALINA_OPTS -Dhttpfs.ssl.enabled "-Dhttpfs.ssl.enabled=${HTTPFS_SSL_ENABLED}"
+
+exec "${HADOOP_CATALINA_HOME}/bin/catalina.sh" "$@"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c4f76aa/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml
deleted file mode 100644
index c91c2e2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml
+++ /dev/null
@@ -1,135 +0,0 @@
-<?xml version='1.0' encoding='utf-8'?>
-<!--
-
- All Rights Reserved.
-
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<!-- Note: A "Server" is not itself a "Container", so you may not
- define subcomponents such as "Valves" at this level.
- Documentation at /docs/config/server.html
- -->
-<Server port="${httpfs.admin.port}" shutdown="SHUTDOWN">
-
- <!--APR library loader. Documentation at /docs/apr.html -->
- <Listener className="org.apache.catalina.core.AprLifecycleListener"
- SSLEngine="on"/>
- <!--Initialize Jasper prior to webapps are loaded. Documentation at /docs/jasper-howto.html -->
- <Listener className="org.apache.catalina.core.JasperListener"/>
- <!-- Prevent memory leaks due to use of particular java/javax APIs-->
- <Listener
- className="org.apache.catalina.core.JreMemoryLeakPreventionListener"/>
- <!-- JMX Support for the Tomcat server. Documentation at /docs/non-existent.html -->
- <Listener className="org.apache.catalina.mbeans.ServerLifecycleListener"/>
- <Listener
- className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener"/>
-
- <!-- Global JNDI resources
- Documentation at /docs/jndi-resources-howto.html
- -->
- <GlobalNamingResources>
- <!-- Editable user database that can also be used by
- UserDatabaseRealm to authenticate users
- -->
- <Resource name="UserDatabase" auth="Container"
- type="org.apache.catalina.UserDatabase"
- description="User database that can be updated and saved"
- factory="org.apache.catalina.users.MemoryUserDatabaseFactory"
- pathname="conf/tomcat-users.xml"/>
- </GlobalNamingResources>
-
- <!-- A "Service" is a collection of one or more "Connectors" that share
- a single "Container" Note: A "Service" is not itself a "Container",
- so you may not define subcomponents such as "Valves" at this level.
- Documentation at /docs/config/service.html
- -->
- <Service name="Catalina">
-
- <!--The connectors can use a shared executor, you can define one or more named thread pools-->
- <!--
- <Executor name="tomcatThreadPool" namePrefix="catalina-exec-"
- maxThreads="150" minSpareThreads="4"/>
- -->
-
- <!-- Define a SSL HTTP/1.1 Connector on port 8443
- This connector uses the JSSE configuration, when using APR, the
- connector should be using the OpenSSL style configuration
- described in the APR documentation -->
- <Connector port="${httpfs.http.port}" protocol="HTTP/1.1" SSLEnabled="true"
- maxThreads="150" scheme="https" secure="true"
- clientAuth="false" sslEnabledProtocols="TLSv1,SSLv2Hello"
- keystoreFile="${httpfs.ssl.keystore.file}"
- keystorePass="${httpfs.ssl.keystore.pass}"/>
-
- <!-- Define an AJP 1.3 Connector on port 8009 -->
-
-
- <!-- An Engine represents the entry point (within Catalina) that processes
- every request. The Engine implementation for Tomcat stand alone
- analyzes the HTTP headers included with the request, and passes them
- on to the appropriate Host (virtual host).
- Documentation at /docs/config/engine.html -->
-
- <!-- You should set jvmRoute to support load-balancing via AJP ie :
- <Engine name="Catalina" defaultHost="localhost" jvmRoute="jvm1">
- -->
- <Engine name="Catalina" defaultHost="localhost">
-
- <!--For clustering, please take a look at documentation at:
- /docs/cluster-howto.html (simple how to)
- /docs/config/cluster.html (reference documentation) -->
- <!--
- <Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"/>
- -->
-
- <!-- The request dumper valve dumps useful debugging information about
- the request and response data received and sent by Tomcat.
- Documentation at: /docs/config/valve.html -->
- <!--
- <Valve className="org.apache.catalina.valves.RequestDumperValve"/>
- -->
-
- <!-- This Realm uses the UserDatabase configured in the global JNDI
- resources under the key "UserDatabase". Any edits
- that are performed against this UserDatabase are immediately
- available for use by the Realm. -->
- <Realm className="org.apache.catalina.realm.UserDatabaseRealm"
- resourceName="UserDatabase"/>
-
- <!-- Define the default virtual host
- Note: XML Schema validation will not work with Xerces 2.2.
- -->
- <Host name="localhost" appBase="webapps"
- unpackWARs="true" autoDeploy="true"
- xmlValidation="false" xmlNamespaceAware="false">
-
- <!-- SingleSignOn valve, share authentication between web applications
- Documentation at: /docs/config/valve.html -->
- <!--
- <Valve className="org.apache.catalina.authenticator.SingleSignOn" />
- -->
-
- <!-- Access log processes all example.
- Documentation at: /docs/config/valve.html -->
- <!--
- <Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
- prefix="localhost_access_log." suffix=".txt" pattern="common" resolveHosts="false"/>
- -->
-
- </Host>
- </Engine>
- </Service>
-</Server>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c4f76aa/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml.conf
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml.conf b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml.conf
new file mode 100644
index 0000000..4a90532
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/tomcat/ssl-server.xml.conf
@@ -0,0 +1,135 @@
+<?xml version='1.0' encoding='utf-8'?>
+<!--
+
+ All Rights Reserved.
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- Note: A "Server" is not itself a "Container", so you may not
+ define subcomponents such as "Valves" at this level.
+ Documentation at /docs/config/server.html
+ -->
+<Server port="${httpfs.admin.port}" shutdown="SHUTDOWN">
+
+ <!--APR library loader. Documentation at /docs/apr.html -->
+ <Listener className="org.apache.catalina.core.AprLifecycleListener"
+ SSLEngine="on"/>
+ <!--Initialize Jasper prior to webapps are loaded. Documentation at /docs/jasper-howto.html -->
+ <Listener className="org.apache.catalina.core.JasperListener"/>
+ <!-- Prevent memory leaks due to use of particular java/javax APIs-->
+ <Listener
+ className="org.apache.catalina.core.JreMemoryLeakPreventionListener"/>
+ <!-- JMX Support for the Tomcat server. Documentation at /docs/non-existent.html -->
+ <Listener className="org.apache.catalina.mbeans.ServerLifecycleListener"/>
+ <Listener
+ className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener"/>
+
+ <!-- Global JNDI resources
+ Documentation at /docs/jndi-resources-howto.html
+ -->
+ <GlobalNamingResources>
+ <!-- Editable user database that can also be used by
+ UserDatabaseRealm to authenticate users
+ -->
+ <Resource name="UserDatabase" auth="Container"
+ type="org.apache.catalina.UserDatabase"
+ description="User database that can be updated and saved"
+ factory="org.apache.catalina.users.MemoryUserDatabaseFactory"
+ pathname="conf/tomcat-users.xml"/>
+ </GlobalNamingResources>
+
+ <!-- A "Service" is a collection of one or more "Connectors" that share
+ a single "Container" Note: A "Service" is not itself a "Container",
+ so you may not define subcomponents such as "Valves" at this level.
+ Documentation at /docs/config/service.html
+ -->
+ <Service name="Catalina">
+
+ <!--The connectors can use a shared executor, you can define one or more named thread pools-->
+ <!--
+ <Executor name="tomcatThreadPool" namePrefix="catalina-exec-"
+ maxThreads="httpfs.max.threads" minSpareThreads="4"/>
+ -->
+
+ <!-- Define a SSL HTTP/1.1 Connector on port 8443
+ This connector uses the JSSE configuration, when using APR, the
+ connector should be using the OpenSSL style configuration
+ described in the APR documentation -->
+ <Connector port="${httpfs.http.port}" protocol="HTTP/1.1" SSLEnabled="true"
+ maxThreads="150" scheme="https" secure="true"
+ clientAuth="false" sslEnabledProtocols="TLSv1,SSLv2Hello"
+ keystoreFile="${httpfs.ssl.keystore.file}"
+ keystorePass="_httpfs_ssl_keystore_pass_"/>
+
+ <!-- Define an AJP 1.3 Connector on port 8009 -->
+
+
+ <!-- An Engine represents the entry point (within Catalina) that processes
+ every request. The Engine implementation for Tomcat stand alone
+ analyzes the HTTP headers included with the request, and passes them
+ on to the appropriate Host (virtual host).
+ Documentation at /docs/config/engine.html -->
+
+ <!-- You should set jvmRoute to support load-balancing via AJP ie :
+ <Engine name="Catalina" defaultHost="localhost" jvmRoute="jvm1">
+ -->
+ <Engine name="Catalina" defaultHost="localhost">
+
+ <!--For clustering, please take a look at documentation at:
+ /docs/cluster-howto.html (simple how to)
+ /docs/config/cluster.html (reference documentation) -->
+ <!--
+ <Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"/>
+ -->
+
+ <!-- The request dumper valve dumps useful debugging information about
+ the request and response data received and sent by Tomcat.
+ Documentation at: /docs/config/valve.html -->
+ <!--
+ <Valve className="org.apache.catalina.valves.RequestDumperValve"/>
+ -->
+
+ <!-- This Realm uses the UserDatabase configured in the global JNDI
+ resources under the key "UserDatabase". Any edits
+ that are performed against this UserDatabase are immediately
+ available for use by the Realm. -->
+ <Realm className="org.apache.catalina.realm.UserDatabaseRealm"
+ resourceName="UserDatabase"/>
+
+ <!-- Define the default virtual host
+ Note: XML Schema validation will not work with Xerces 2.2.
+ -->
+ <Host name="localhost" appBase="webapps"
+ unpackWARs="true" autoDeploy="true"
+ xmlValidation="false" xmlNamespaceAware="false">
+
+ <!-- SingleSignOn valve, share authentication between web applications
+ Documentation at: /docs/config/valve.html -->
+ <!--
+ <Valve className="org.apache.catalina.authenticator.SingleSignOn" />
+ -->
+
+ <!-- Access log processes all example.
+ Documentation at: /docs/config/valve.html -->
+ <!--
+ <Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
+ prefix="localhost_access_log." suffix=".txt" pattern="common" resolveHosts="false"/>
+ -->
+
+ </Host>
+ </Engine>
+ </Service>
+</Server>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c4f76aa/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/ServerSetup.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/ServerSetup.apt.vm b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/ServerSetup.apt.vm
deleted file mode 100644
index 878ab1f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/ServerSetup.apt.vm
+++ /dev/null
@@ -1,159 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License.
-
- ---
- Hadoop HDFS over HTTP ${project.version} - Server Setup
- ---
- ---
- ${maven.build.timestamp}
-
-Hadoop HDFS over HTTP ${project.version} - Server Setup
-
- This page explains how to quickly setup HttpFS with Pseudo authentication
- against a Hadoop cluster with Pseudo authentication.
-
-* Requirements
-
- * Java 6+
-
- * Maven 3+
-
-* Install HttpFS
-
-+---+
-~ $ tar xzf httpfs-${project.version}.tar.gz
-+---+
-
-* Configure HttpFS
-
- By default, HttpFS assumes that Hadoop configuration files
- (<<<core-site.xml & hdfs-site.xml>>>) are in the HttpFS
- configuration directory.
-
- If this is not the case, add to the <<<httpfs-site.xml>>> file the
- <<<httpfs.hadoop.config.dir>>> property set to the location
- of the Hadoop configuration directory.
-
-* Configure Hadoop
-
- Edit Hadoop <<<core-site.xml>>> and defined the Unix user that will
- run the HttpFS server as a proxyuser. For example:
-
-+---+
- ...
- <property>
- <name>hadoop.proxyuser.#HTTPFSUSER#.hosts</name>
- <value>httpfs-host.foo.com</value>
- </property>
- <property>
- <name>hadoop.proxyuser.#HTTPFSUSER#.groups</name>
- <value>*</value>
- </property>
- ...
-+---+
-
- IMPORTANT: Replace <<<#HTTPFSUSER#>>> with the Unix user that will
- start the HttpFS server.
-
-* Restart Hadoop
-
- You need to restart Hadoop for the proxyuser configuration ot become
- active.
-
-* Start/Stop HttpFS
-
- To start/stop HttpFS use HttpFS's bin/httpfs.sh script. For example:
-
-+---+
-httpfs-${project.version} $ bin/httpfs.sh start
-+---+
-
- NOTE: Invoking the script without any parameters list all possible
- parameters (start, stop, run, etc.). The <<<httpfs.sh>>> script is a wrapper
- for Tomcat's <<<catalina.sh>>> script that sets the environment variables
- and Java System properties required to run HttpFS server.
-
-* Test HttpFS is working
-
-+---+
-~ $ curl -i "http://<HTTPFSHOSTNAME>:14000?user.name=babu&op=homedir"
-HTTP/1.1 200 OK
-Content-Type: application/json
-Transfer-Encoding: chunked
-
-{"homeDir":"http:\/\/<HTTPFS_HOST>:14000\/user\/babu"}
-+---+
-
-* Embedded Tomcat Configuration
-
- To configure the embedded Tomcat go to the <<<tomcat/conf>>>.
-
- HttpFS preconfigures the HTTP and Admin ports in Tomcat's <<<server.xml>>> to
- 14000 and 14001.
-
- Tomcat logs are also preconfigured to go to HttpFS's <<<logs/>>> directory.
-
- The following environment variables (which can be set in HttpFS's
- <<<conf/httpfs-env.sh>>> script) can be used to alter those values:
-
- * HTTPFS_HTTP_PORT
-
- * HTTPFS_ADMIN_PORT
-
- * HTTPFS_LOG
-
-* HttpFS Configuration
-
- HttpFS supports the following {{{./httpfs-default.html}configuration properties}}
- in the HttpFS's <<<conf/httpfs-site.xml>>> configuration file.
-
-* HttpFS over HTTPS (SSL)
-
- To configure HttpFS to work over SSL edit the {{httpfs-env.sh}} script in the
- configuration directory setting the {{HTTPFS_SSL_ENABLED}} to {{true}}.
-
- In addition, the following 2 properties may be defined (shown with default
- values):
-
- * HTTPFS_SSL_KEYSTORE_FILE=${HOME}/.keystore
-
- * HTTPFS_SSL_KEYSTORE_PASS=password
-
- In the HttpFS <<<tomcat/conf>>> directory, replace the <<<server.xml>>> file
- with the <<<ssl-server.xml>>> file.
-
-
- You need to create an SSL certificate for the HttpFS server. As the
- <<<httpfs>>> Unix user, using the Java <<<keytool>>> command to create the
- SSL certificate:
-
-+---+
-$ keytool -genkey -alias tomcat -keyalg RSA
-+---+
-
- You will be asked a series of questions in an interactive prompt. It will
- create the keystore file, which will be named <<.keystore>> and located in the
- <<<httpfs>>> user home directory.
-
- The password you enter for "keystore password" must match the value of the
- <<<HTTPFS_SSL_KEYSTORE_PASS>>> environment variable set in the
- <<<httpfs-env.sh>>> script in the configuration directory.
-
- The answer to "What is your first and last name?" (i.e. "CN") must be the
- hostname of the machine where the HttpFS Server will be running.
-
- Start HttpFS. It should work over HTTPS.
-
- Using the Hadoop <<<FileSystem>>> API or the Hadoop FS shell, use the
- <<<swebhdfs://>>> scheme. Make sure the JVM is picking up the truststore
- containing the public key of the SSL certificate if using a self-signed
- certificate.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c4f76aa/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/UsingHttpTools.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/UsingHttpTools.apt.vm b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/UsingHttpTools.apt.vm
deleted file mode 100644
index c93e20b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/UsingHttpTools.apt.vm
+++ /dev/null
@@ -1,87 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License.
-
- ---
- Hadoop HDFS over HTTP ${project.version} - Using HTTP Tools
- ---
- ---
- ${maven.build.timestamp}
-
-Hadoop HDFS over HTTP ${project.version} - Using HTTP Tools
-
-* Security
-
- Out of the box HttpFS supports both pseudo authentication and Kerberos HTTP
- SPNEGO authentication.
-
-** Pseudo Authentication
-
- With pseudo authentication the user name must be specified in the
- <<<user.name=\<USERNAME\>>>> query string parameter of a HttpFS URL.
- For example:
-
-+---+
-$ curl "http://<HTTFS_HOST>:14000/webhdfs/v1?op=homedir&user.name=babu"
-+---+
-
-** Kerberos HTTP SPNEGO Authentication
-
- Kerberos HTTP SPNEGO authentication requires a tool or library supporting
- Kerberos HTTP SPNEGO protocol.
-
- IMPORTANT: If using <<<curl>>>, the <<<curl>>> version being used must support
- GSS (<<<curl -V>>> prints out 'GSS' if it supports it).
-
- For example:
-
-+---+
-$ kinit
-Please enter the password for tucu@LOCALHOST:
-$ curl --negotiate -u foo "http://<HTTPFS_HOST>:14000/webhdfs/v1?op=homedir"
-Enter host password for user 'foo':
-+---+
-
- NOTE: the <<<-u USER>>> option is required by the <<<--negotiate>>> but it is
- not used. Use any value as <<<USER>>> and when asked for the password press
- [ENTER] as the password value is ignored.
-
-** {Remembering Who I Am} (Establishing an Authenticated Session)
-
- As most authentication mechanisms, Hadoop HTTP authentication authenticates
- users once and issues a short-lived authentication token to be presented in
- subsequent requests. This authentication token is a signed HTTP Cookie.
-
- When using tools like <<<curl>>>, the authentication token must be stored on
- the first request doing authentication, and submitted in subsequent requests.
- To do this with curl the <<<-b>>> and <<<-c>>> options to save and send HTTP
- Cookies must be used.
-
- For example, the first request doing authentication should save the received
- HTTP Cookies.
-
- Using Pseudo Authentication:
-
-+---+
-$ curl -c ~/.httpfsauth "http://<HTTPFS_HOST>:14000/webhdfs/v1?op=homedir&user.name=babu"
-+---+
-
- Using Kerberos HTTP SPNEGO authentication:
-
-+---+
-$ curl --negotiate -u foo -c ~/.httpfsauth "http://<HTTPFS_HOST>:14000/webhdfs/v1?op=homedir"
-+---+
-
- Then, subsequent requests forward the previously received HTTP Cookie:
-
-+---+
-$ curl -b ~/.httpfsauth "http://<HTTPFS_HOST>:14000/webhdfs/v1?op=liststatus"
-+---+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c4f76aa/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/index.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/index.apt.vm b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/index.apt.vm
deleted file mode 100644
index f51e743..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/apt/index.apt.vm
+++ /dev/null
@@ -1,83 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License.
-
- ---
- Hadoop HDFS over HTTP - Documentation Sets ${project.version}
- ---
- ---
- ${maven.build.timestamp}
-
-Hadoop HDFS over HTTP - Documentation Sets ${project.version}
-
- HttpFS is a server that provides a REST HTTP gateway supporting all HDFS
- File System operations (read and write). And it is inteoperable with the
- <<webhdfs>> REST HTTP API.
-
- HttpFS can be used to transfer data between clusters running different
- versions of Hadoop (overcoming RPC versioning issues), for example using
- Hadoop DistCP.
-
- HttpFS can be used to access data in HDFS on a cluster behind of a firewall
- (the HttpFS server acts as a gateway and is the only system that is allowed
- to cross the firewall into the cluster).
-
- HttpFS can be used to access data in HDFS using HTTP utilities (such as curl
- and wget) and HTTP libraries Perl from other languages than Java.
-
- The <<webhdfs>> client FileSytem implementation can be used to access HttpFS
- using the Hadoop filesystem command (<<<hadoop fs>>>) line tool as well as
- from Java aplications using the Hadoop FileSystem Java API.
-
- HttpFS has built-in security supporting Hadoop pseudo authentication and
- HTTP SPNEGO Kerberos and other pluggable authentication mechanims. It also
- provides Hadoop proxy user support.
-
-* How Does HttpFS Works?
-
- HttpFS is a separate service from Hadoop NameNode.
-
- HttpFS itself is Java web-application and it runs using a preconfigured Tomcat
- bundled with HttpFS binary distribution.
-
- HttpFS HTTP web-service API calls are HTTP REST calls that map to a HDFS file
- system operation. For example, using the <<<curl>>> Unix command:
-
- * <<<$ curl http://httpfs-host:14000/webhdfs/v1/user/foo/README.txt>>> returns
- the contents of the HDFS <<</user/foo/README.txt>>> file.
-
- * <<<$ curl http://httpfs-host:14000/webhdfs/v1/user/foo?op=list>>> returns the
- contents of the HDFS <<</user/foo>>> directory in JSON format.
-
- * <<<$ curl -X POST http://httpfs-host:14000/webhdfs/v1/user/foo/bar?op=mkdirs>>>
- creates the HDFS <<</user/foo.bar>>> directory.
-
-* How HttpFS and Hadoop HDFS Proxy differ?
-
- HttpFS was inspired by Hadoop HDFS proxy.
-
- HttpFS can be seen as a full rewrite of Hadoop HDFS proxy.
-
- Hadoop HDFS proxy provides a subset of file system operations (read only),
- HttpFS provides support for all file system operations.
-
- HttpFS uses a clean HTTP REST API making its use with HTTP tools more
- intuitive.
-
- HttpFS supports Hadoop pseudo authentication, Kerberos SPNEGOS authentication
- and Hadoop proxy users. Hadoop HDFS proxy did not.
-
-* User and Developer Documentation
-
- * {{{./ServerSetup.html}HttpFS Server Setup}}
-
- * {{{./UsingHttpTools.html}Using HTTP Tools}}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c4f76aa/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
new file mode 100644
index 0000000..3c7f9d3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/ServerSetup.md.vm
@@ -0,0 +1,121 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Hadoop HDFS over HTTP - Server Setup
+====================================
+
+This page explains how to quickly setup HttpFS with Pseudo authentication against a Hadoop cluster with Pseudo authentication.
+
+Install HttpFS
+--------------
+
+ ~ $ tar xzf httpfs-${project.version}.tar.gz
+
+Configure HttpFS
+----------------
+
+By default, HttpFS assumes that Hadoop configuration files (`core-site.xml & hdfs-site.xml`) are in the HttpFS configuration directory.
+
+If this is not the case, add to the `httpfs-site.xml` file the `httpfs.hadoop.config.dir` property set to the location of the Hadoop configuration directory.
+
+Configure Hadoop
+----------------
+
+Edit Hadoop `core-site.xml` and defined the Unix user that will run the HttpFS server as a proxyuser. For example:
+
+```xml
+ <property>
+ <name>hadoop.proxyuser.#HTTPFSUSER#.hosts</name>
+ <value>httpfs-host.foo.com</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.#HTTPFSUSER#.groups</name>
+ <value>*</value>
+ </property>
+```
+
+IMPORTANT: Replace `#HTTPFSUSER#` with the Unix user that will start the HttpFS server.
+
+Restart Hadoop
+--------------
+
+You need to restart Hadoop for the proxyuser configuration ot become active.
+
+Start/Stop HttpFS
+-----------------
+
+To start/stop HttpFS use HttpFS's sbin/httpfs.sh script. For example:
+
+ $ sbin/httpfs.sh start
+
+NOTE: Invoking the script without any parameters list all possible parameters (start, stop, run, etc.). The `httpfs.sh` script is a wrapper for Tomcat's `catalina.sh` script that sets the environment variables and Java System properties required to run HttpFS server.
+
+Test HttpFS is working
+----------------------
+
+ ~ $ curl -i "http://<HTTPFSHOSTNAME>:14000?user.name=babu&op=homedir"
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+
+ {"homeDir":"http:\/\/<HTTPFS_HOST>:14000\/user\/babu"}
+
+Embedded Tomcat Configuration
+-----------------------------
+
+To configure the embedded Tomcat go to the `tomcat/conf`.
+
+HttpFS preconfigures the HTTP and Admin ports in Tomcat's `server.xml` to 14000 and 14001.
+
+Tomcat logs are also preconfigured to go to HttpFS's `logs/` directory.
+
+The following environment variables (which can be set in HttpFS's `etc/hadoop/httpfs-env.sh` script) can be used to alter those values:
+
+* HTTPFS\_HTTP\_PORT
+
+* HTTPFS\_ADMIN\_PORT
+
+* HADOOP\_LOG\_DIR
+
+HttpFS Configuration
+--------------------
+
+HttpFS supports the following [configuration properties](./httpfs-default.html) in the HttpFS's `etc/hadoop/httpfs-site.xml` configuration file.
+
+HttpFS over HTTPS (SSL)
+-----------------------
+
+To configure HttpFS to work over SSL edit the [httpfs-env.sh](#httpfs-env.sh) script in the configuration directory setting the [HTTPFS\_SSL\_ENABLED](#HTTPFS_SSL_ENABLED) to [true](#true).
+
+In addition, the following 2 properties may be defined (shown with default values):
+
+* HTTPFS\_SSL\_KEYSTORE\_FILE=$HOME/.keystore
+
+* HTTPFS\_SSL\_KEYSTORE\_PASS=password
+
+In the HttpFS `tomcat/conf` directory, replace the `server.xml` file with the `ssl-server.xml` file.
+
+You need to create an SSL certificate for the HttpFS server. As the `httpfs` Unix user, using the Java `keytool` command to create the SSL certificate:
+
+ $ keytool -genkey -alias tomcat -keyalg RSA
+
+You will be asked a series of questions in an interactive prompt. It will create the keystore file, which will be named **.keystore** and located in the `httpfs` user home directory.
+
+The password you enter for "keystore password" must match the value of the `HTTPFS_SSL_KEYSTORE_PASS` environment variable set in the `httpfs-env.sh` script in the configuration directory.
+
+The answer to "What is your first and last name?" (i.e. "CN") must be the hostname of the machine where the HttpFS Server will be running.
+
+Start HttpFS. It should work over HTTPS.
+
+Using the Hadoop `FileSystem` API or the Hadoop FS shell, use the `swebhdfs://` scheme. Make sure the JVM is picking up the truststore containing the public key of the SSL certificate if using a self-signed certificate.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c4f76aa/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/UsingHttpTools.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/UsingHttpTools.md b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/UsingHttpTools.md
new file mode 100644
index 0000000..3045ad6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/UsingHttpTools.md
@@ -0,0 +1,62 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Hadoop HDFS over HTTP - Using HTTP Tools
+========================================
+
+Security
+--------
+
+Out of the box HttpFS supports both pseudo authentication and Kerberos HTTP SPNEGO authentication.
+
+### Pseudo Authentication
+
+With pseudo authentication the user name must be specified in the `user.name=<USERNAME>` query string parameter of a HttpFS URL. For example:
+
+ $ curl "http://<HTTFS_HOST>:14000/webhdfs/v1?op=homedir&user.name=babu"
+
+### Kerberos HTTP SPNEGO Authentication
+
+Kerberos HTTP SPNEGO authentication requires a tool or library supporting Kerberos HTTP SPNEGO protocol.
+
+IMPORTANT: If using `curl`, the `curl` version being used must support GSS (`curl -V` prints out 'GSS' if it supports it).
+
+For example:
+
+ $ kinit
+ Please enter the password for user@LOCALHOST:
+ $ curl --negotiate -u foo "http://<HTTPFS_HOST>:14000/webhdfs/v1?op=homedir"
+ Enter host password for user 'foo':
+
+NOTE: the `-u USER` option is required by the `--negotiate` but it is not used. Use any value as `USER` and when asked for the password press [ENTER] as the password value is ignored.
+
+### Remembering Who I Am (Establishing an Authenticated Session)
+
+As most authentication mechanisms, Hadoop HTTP authentication authenticates users once and issues a short-lived authentication token to be presented in subsequent requests. This authentication token is a signed HTTP Cookie.
+
+When using tools like `curl`, the authentication token must be stored on the first request doing authentication, and submitted in subsequent requests. To do this with curl the `-b` and `-c` options to save and send HTTP Cookies must be used.
+
+For example, the first request doing authentication should save the received HTTP Cookies.
+
+Using Pseudo Authentication:
+
+ $ curl -c ~/.httpfsauth "http://<HTTPFS_HOST>:14000/webhdfs/v1?op=homedir&user.name=foo"
+
+Using Kerberos HTTP SPNEGO authentication:
+
+ $ curl --negotiate -u foo -c ~/.httpfsauth "http://<HTTPFS_HOST>:14000/webhdfs/v1?op=homedir"
+
+Then, subsequent requests forward the previously received HTTP Cookie:
+
+ $ curl -b ~/.httpfsauth "http://<HTTPFS_HOST>:14000/webhdfs/v1?op=liststatus"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c4f76aa/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md
new file mode 100644
index 0000000..ac8f4ca
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md
@@ -0,0 +1,52 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Hadoop HDFS over HTTP - Documentation Sets
+==========================================
+
+HttpFS is a server that provides a REST HTTP gateway supporting all HDFS File System operations (read and write). And it is inteoperable with the **webhdfs** REST HTTP API.
+
+HttpFS can be used to transfer data between clusters running different versions of Hadoop (overcoming RPC versioning issues), for example using Hadoop DistCP.
+
+HttpFS can be used to access data in HDFS on a cluster behind of a firewall (the HttpFS server acts as a gateway and is the only system that is allowed to cross the firewall into the cluster).
+
+HttpFS can be used to access data in HDFS using HTTP utilities (such as curl and wget) and HTTP libraries Perl from other languages than Java.
+
+The **webhdfs** client FileSytem implementation can be used to access HttpFS using the Hadoop filesystem command (`hadoop fs`) line tool as well as from Java aplications using the Hadoop FileSystem Java API.
+
+HttpFS has built-in security supporting Hadoop pseudo authentication and HTTP SPNEGO Kerberos and other pluggable authentication mechanims. It also provides Hadoop proxy user support.
+
+How Does HttpFS Works?
+----------------------
+
+HttpFS is a separate service from Hadoop NameNode.
+
+HttpFS itself is Java web-application and it runs using a preconfigured Tomcat bundled with HttpFS binary distribution.
+
+HttpFS HTTP web-service API calls are HTTP REST calls that map to a HDFS file system operation. For example, using the `curl` Unix command:
+
+* `$ curl http://httpfs-host:14000/webhdfs/v1/user/foo/README.txt` returns the contents of the HDFS `/user/foo/README.txt` file.
+
+* `$ curl http://httpfs-host:14000/webhdfs/v1/user/foo?op=list` returns the contents of the HDFS `/user/foo` directory in JSON format.
+
+* `$ curl -X POST http://httpfs-host:14000/webhdfs/v1/user/foo/bar?op=mkdirs` creates the HDFS `/user/foo.bar` directory.
+
+User and Developer Documentation
+--------------------------------
+
+* [HttpFS Server Setup](./ServerSetup.html)
+
+* [Using HTTP Tools](./UsingHttpTools.html)
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c4f76aa/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f8b0c37..6dc7a0f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -141,6 +141,8 @@ Trunk (Unreleased)
HDFS-7668. Convert site documentation from apt to markdown (Masatake
Iwasaki via aw)
+ HDFS-7460. Rewrite httpfs to use new shell framework (John Smith via aw)
+
OPTIMIZATIONS
BUG FIXES
[02/17] hadoop git commit: HDFS-7495. Remove updatePosition argument
from DFSInputStream#getBlockAt() (cmccabe)
Posted by zj...@apache.org.
HDFS-7495. Remove updatePosition argument from DFSInputStream#getBlockAt() (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/caa42adf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/caa42adf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/caa42adf
Branch: refs/heads/YARN-2928
Commit: caa42adf208bfb5625d1b3ef665fbf334ffcccd9
Parents: 5731c0e
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Wed Feb 25 13:29:31 2015 -0800
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Wed Feb 25 13:30:01 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../org/apache/hadoop/hdfs/DFSInputStream.java | 33 ++++++++------------
2 files changed, 16 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/caa42adf/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4ca10da..e0f9267 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -668,6 +668,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7740. Test truncate with DataNodes restarting. (yliu)
+ HDFS-7495. Remove updatePosition argument from DFSInputStream#getBlockAt()
+ (cmccabe)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/caa42adf/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index c408524..cf8015f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -434,12 +434,10 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
* Fetch it from the namenode if not cached.
*
* @param offset block corresponding to this offset in file is returned
- * @param updatePosition whether to update current position
* @return located block
* @throws IOException
*/
- private LocatedBlock getBlockAt(long offset,
- boolean updatePosition) throws IOException {
+ private LocatedBlock getBlockAt(long offset) throws IOException {
synchronized(infoLock) {
assert (locatedBlocks != null) : "locatedBlocks is null";
@@ -449,7 +447,6 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
if (offset < 0 || offset >= getFileLength()) {
throw new IOException("offset < 0 || offset >= getFileLength(), offset="
+ offset
- + ", updatePosition=" + updatePosition
+ ", locatedBlocks=" + locatedBlocks);
}
else if (offset >= locatedBlocks.getFileLength()) {
@@ -470,17 +467,6 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
}
blk = locatedBlocks.get(targetBlockIdx);
}
-
- // update current position
- if (updatePosition) {
- // synchronized not strictly needed, since we only get here
- // from synchronized caller methods
- synchronized(this) {
- pos = offset;
- blockEnd = blk.getStartOffset() + blk.getBlockSize() - 1;
- currentLocatedBlock = blk;
- }
- }
return blk;
}
}
@@ -604,7 +590,14 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
//
// Compute desired block
//
- LocatedBlock targetBlock = getBlockAt(target, true);
+ LocatedBlock targetBlock = getBlockAt(target);
+
+ // update current position
+ this.pos = target;
+ this.blockEnd = targetBlock.getStartOffset() +
+ targetBlock.getBlockSize() - 1;
+ this.currentLocatedBlock = targetBlock;
+
assert (target==pos) : "Wrong postion " + pos + " expect " + target;
long offsetIntoBlock = target - targetBlock.getStartOffset();
@@ -979,7 +972,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
}
deadNodes.clear(); //2nd option is to remove only nodes[blockId]
openInfo();
- block = getBlockAt(block.getStartOffset(), false);
+ block = getBlockAt(block.getStartOffset());
failures++;
continue;
}
@@ -1056,7 +1049,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
byte[] buf, int offset,
Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap)
throws IOException {
- block = getBlockAt(block.getStartOffset(), false);
+ block = getBlockAt(block.getStartOffset());
while (true) {
DNAddrPair addressPair = chooseDataNode(block, null);
try {
@@ -1108,7 +1101,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
// start of the loop.
CachingStrategy curCachingStrategy;
boolean allowShortCircuitLocalReads;
- block = getBlockAt(block.getStartOffset(), false);
+ block = getBlockAt(block.getStartOffset());
synchronized(infoLock) {
curCachingStrategy = cachingStrategy;
allowShortCircuitLocalReads = !shortCircuitForbidden();
@@ -1208,7 +1201,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
ByteBuffer bb = null;
int len = (int) (end - start + 1);
int hedgedReadId = 0;
- block = getBlockAt(block.getStartOffset(), false);
+ block = getBlockAt(block.getStartOffset());
while (true) {
// see HDFS-6591, this metric is used to verify/catch unnecessary loops
hedgedReadOpsLoopNumForTesting++;
[11/17] hadoop git commit: HADOOP-11637. bash location hard-coded in
shell scripts (aw)
Posted by zj...@apache.org.
HADOOP-11637. bash location hard-coded in shell scripts (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dce8b9c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dce8b9c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dce8b9c4
Branch: refs/heads/YARN-2928
Commit: dce8b9c4d0b2da1780f743d81e840ca0fdfc62cf
Parents: 773b651
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Feb 26 09:29:16 2015 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Feb 26 09:29:16 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../hadoop-common/src/main/bin/hadoop-functions.sh | 2 +-
.../hadoop-common/src/site/markdown/RackAwareness.md | 5 +++--
hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh | 2 +-
hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh | 2 +-
hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh | 2 +-
.../hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh | 2 +-
.../hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh | 2 +-
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 2 +-
hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh | 2 +-
hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 2 +-
11 files changed, 14 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dce8b9c4/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 39062a8..ca27463 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -404,6 +404,8 @@ Trunk (Unreleased)
HADOOP-11625. Minor fixes to command manual & SLA doc (aw)
+ HADOOP-11637. bash location hard-coded in shell scripts (aw)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dce8b9c4/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index cec6b2c..bccbe25 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dce8b9c4/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md b/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
index c5ab19a..09f5610 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
@@ -105,7 +105,7 @@ bash Example
------------
```bash
-#!/bin/bash
+#!/usr/bin/env bash
# Here's a bash example to show just how simple these scripts can be
# Assuming we have flat network with everything on a single switch, we can fake a rack topology.
# This could occur in a lab environment where we have limited nodes,like 2-8 physical machines on a unmanaged switch.
@@ -133,4 +133,5 @@ bash Example
# fails to split on four dots, it will still print '/rack-' last field value
echo $@ | xargs -n 1 | awk -F '.' '{print "/rack-"$NF}'
-```
\ No newline at end of file
+```
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dce8b9c4/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh b/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
index de9554f..41449ef 100644
--- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
+++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dce8b9c4/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh b/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh
index c1aa136..df48889 100644
--- a/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh
+++ b/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dce8b9c4/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh b/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh
index e4d4f93..9228d2d 100644
--- a/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh
+++ b/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dce8b9c4/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
index 0e8cc40..a4edef6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/conf/httpfs-env.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dce8b9c4/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
index e520e08..767bd6e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dce8b9c4/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
index 72dca93..f51a5e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dce8b9c4/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh b/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh
index c8e38ac..f4fdabd 100644
--- a/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh
+++ b/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dce8b9c4/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
index 463ae23..d1409bf 100644
--- a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
+++ b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.