You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/05/22 23:41:19 UTC
[01/50] [abbrv] hadoop git commit: YARN-3505 addendum: fix an issue
in previous patch.
Repository: hadoop
Updated Branches:
refs/heads/YARN-2928 827633ee9 -> e19566a66
YARN-3505 addendum: fix an issue in previous patch.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aacec998
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aacec998
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aacec998
Branch: refs/heads/YARN-2928
Commit: aacec9983b369ca1d1225f0b69b2b455deec677b
Parents: 1157c75
Author: Junping Du <ju...@apache.org>
Authored: Fri May 15 06:39:39 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:46:59 2015 -0700
----------------------------------------------------------------------
.../containermanager/logaggregation/AppLogAggregatorImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aacec998/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index dd2ab25..81be813 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -359,7 +359,7 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
finalReport.setApplicationId(appId);
finalReport.setLogAggregationStatus(renameTemporaryLogFileFailed
? LogAggregationStatus.FAILED : LogAggregationStatus.SUCCEEDED);
- this.context.getLogAggregationStatusForApps().add(report);
+ this.context.getLogAggregationStatusForApps().add(finalReport);
}
} finally {
if (writer != null) {
[16/50] [abbrv] hadoop git commit: HDFS-8345. Storage policy APIs
must be exposed via the FileSystem interface. (Arpit Agarwal)
Posted by zj...@apache.org.
HDFS-8345. Storage policy APIs must be exposed via the FileSystem interface. (Arpit Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6a120187
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6a120187
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6a120187
Branch: refs/heads/YARN-2928
Commit: 6a1201872925face7ca7dc4e18d97fda65c63aea
Parents: 2197cd5
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon May 18 11:36:29 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:47:03 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/fs/AbstractFileSystem.java | 27 ++++++++
.../apache/hadoop/fs/BlockStoragePolicySpi.java | 72 ++++++++++++++++++++
.../java/org/apache/hadoop/fs/FileContext.java | 33 +++++++++
.../java/org/apache/hadoop/fs/FileSystem.java | 28 ++++++++
.../org/apache/hadoop/fs/FilterFileSystem.java | 13 ++++
.../java/org/apache/hadoop/fs/FilterFs.java | 13 ++++
.../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 14 ++++
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 14 ++++
.../org/apache/hadoop/fs/TestHarFileSystem.java | 7 ++
.../hdfs/protocol/BlockStoragePolicy.java | 8 ++-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../main/java/org/apache/hadoop/fs/Hdfs.java | 13 ++++
.../hadoop/hdfs/DistributedFileSystem.java | 27 +++++---
.../apache/hadoop/hdfs/server/mover/Mover.java | 5 +-
.../hadoop/hdfs/tools/StoragePolicyAdmin.java | 5 +-
.../hadoop/hdfs/TestBlockStoragePolicy.java | 64 ++++++++++-------
16 files changed, 308 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a120187/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
index 7af5fa7..cb3fb86 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
@@ -23,6 +23,7 @@ import java.lang.reflect.Constructor;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
@@ -1221,6 +1222,32 @@ public abstract class AbstractFileSystem {
+ " doesn't support deleteSnapshot");
}
+ /**
+ * Set the storage policy for a given file or directory.
+ *
+ * @param path file or directory path.
+ * @param policyName the name of the target storage policy. The list
+ * of supported Storage policies can be retrieved
+ * via {@link #getAllStoragePolicies}.
+ */
+ public void setStoragePolicy(final Path path, final String policyName)
+ throws IOException {
+ throw new UnsupportedOperationException(getClass().getSimpleName()
+ + " doesn't support setStoragePolicy");
+ }
+
+ /**
+ * Retrieve all the storage policies supported by this file system.
+ *
+ * @return all storage policies supported by this filesystem.
+ * @throws IOException
+ */
+ public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
+ throws IOException {
+ throw new UnsupportedOperationException(getClass().getSimpleName()
+ + " doesn't support getAllStoragePolicies");
+ }
+
@Override //Object
public int hashCode() {
return myUri.hashCode();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a120187/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockStoragePolicySpi.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockStoragePolicySpi.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockStoragePolicySpi.java
new file mode 100644
index 0000000..1d6502e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockStoragePolicySpi.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * A storage policy specifies the placement of block replicas on specific
+ * storage types.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public interface BlockStoragePolicySpi {
+
+ /**
+ * Return the name of the storage policy. Policies are uniquely
+ * identified by name.
+ *
+ * @return the name of the storage policy.
+ */
+ String getName();
+
+ /**
+ * Return the preferred storage types associated with this policy. These
+ * storage types are used sequentially for successive block replicas.
+ *
+ * @return preferred storage types used for placing block replicas.
+ */
+ StorageType[] getStorageTypes();
+
+ /**
+ * Get the fallback storage types for creating new block replicas. Fallback
+ * storage types are used if the preferred storage types are not available.
+ *
+ * @return fallback storage types for new block replicas..
+ */
+ StorageType[] getCreationFallbacks();
+
+ /**
+ * Get the fallback storage types for replicating existing block replicas.
+ * Fallback storage types are used if the preferred storage types are not
+ * available.
+ *
+ * @return fallback storage types for replicating existing block replicas.
+ */
+ StorageType[] getReplicationFallbacks();
+
+ /**
+ * Returns true if the policy is inherit-only and cannot be changed for
+ * an existing file.
+ *
+ * @return true if the policy is inherit-only.
+ */
+ boolean isCopyOnCreateFile();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a120187/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index 16cb591..122ddf6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -24,6 +24,7 @@ import java.io.OutputStream;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.IdentityHashMap;
@@ -2665,4 +2666,36 @@ public class FileContext {
}
}.resolve(this, absF);
}
+
+ /**
+ * Set the storage policy for a given file or directory.
+ *
+ * @param path file or directory path.
+ * @param policyName the name of the target storage policy. The list
+ * of supported Storage policies can be retrieved
+ * via {@link #getAllStoragePolicies}.
+ */
+ public void setStoragePolicy(final Path path, final String policyName)
+ throws IOException {
+ final Path absF = fixRelativePart(path);
+ new FSLinkResolver<Void>() {
+ @Override
+ public Void next(final AbstractFileSystem fs, final Path p)
+ throws IOException {
+ fs.setStoragePolicy(path, policyName);
+ return null;
+ }
+ }.resolve(this, absF);
+ }
+
+ /**
+ * Retrieve all the storage policies supported by this file system.
+ *
+ * @return all storage policies supported by this filesystem.
+ * @throws IOException
+ */
+ public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
+ throws IOException {
+ return defaultFS.getAllStoragePolicies();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a120187/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index b508727..33d7c88 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -26,6 +26,7 @@ import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
@@ -2609,6 +2610,33 @@ public abstract class FileSystem extends Configured implements Closeable {
+ " doesn't support removeXAttr");
}
+ /**
+ * Set the storage policy for a given file or directory.
+ *
+ * @param src file or directory path.
+ * @param policyName the name of the target storage policy. The list
+ * of supported Storage policies can be retrieved
+ * via {@link #getAllStoragePolicies}.
+ * @throws IOException
+ */
+ public void setStoragePolicy(final Path src, final String policyName)
+ throws IOException {
+ throw new UnsupportedOperationException(getClass().getSimpleName()
+ + " doesn't support setStoragePolicy");
+ }
+
+ /**
+ * Retrieve all the storage policies supported by this file system.
+ *
+ * @return all storage policies supported by this filesystem.
+ * @throws IOException
+ */
+ public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
+ throws IOException {
+ throw new UnsupportedOperationException(getClass().getSimpleName()
+ + " doesn't support getAllStoragePolicies");
+ }
+
// making it volatile to be able to do a double checked locking
private volatile static boolean FILE_SYSTEMS_LOADED = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a120187/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
index ec056a4..11f3b23 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs;
import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.Collection;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
@@ -619,4 +620,16 @@ public class FilterFileSystem extends FileSystem {
public void removeXAttr(Path path, String name) throws IOException {
fs.removeXAttr(path, name);
}
+
+ @Override
+ public void setStoragePolicy(Path src, String policyName)
+ throws IOException {
+ fs.setStoragePolicy(src, policyName);
+ }
+
+ @Override
+ public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
+ throws IOException {
+ return fs.getAllStoragePolicies();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a120187/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
index 2ba6318..539b26e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
@@ -20,6 +20,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.Collection;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
@@ -397,4 +398,16 @@ public abstract class FilterFs extends AbstractFileSystem {
throws IOException {
myFs.deleteSnapshot(path, snapshotName);
}
+
+ @Override
+ public void setStoragePolicy(Path path, String policyName)
+ throws IOException {
+ myFs.setStoragePolicy(path, policyName);
+ }
+
+ @Override
+ public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
+ throws IOException {
+ return myFs.getAllStoragePolicies();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a120187/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
index a05a700..4e5a0d5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
@@ -20,6 +20,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.Collection;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
@@ -28,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.BlockStoragePolicySpi;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -378,6 +380,18 @@ class ChRootedFs extends AbstractFileSystem {
}
@Override
+ public void setStoragePolicy(Path path, String policyName)
+ throws IOException {
+ myFs.setStoragePolicy(fullPath(path), policyName);
+ }
+
+ @Override
+ public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
+ throws IOException {
+ return myFs.getAllStoragePolicies();
+ }
+
+ @Override
public void setVerifyChecksum(final boolean verifyChecksum)
throws IOException, UnresolvedLinkException {
myFs.setVerifyChecksum(verifyChecksum);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a120187/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
index a23aa86..bec292c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
@@ -740,6 +740,14 @@ public class ViewFs extends AbstractFileSystem {
res.targetFileSystem.deleteSnapshot(res.remainingPath, snapshotName);
}
+ @Override
+ public void setStoragePolicy(final Path path, final String policyName)
+ throws IOException {
+ InodeTree.ResolveResult<AbstractFileSystem> res =
+ fsState.resolve(getUriPath(path), true);
+ res.targetFileSystem.setStoragePolicy(res.remainingPath, policyName);
+ }
+
/*
* An instance of this class represents an internal dir of the viewFs
* ie internal dir of the mount table.
@@ -1070,5 +1078,11 @@ public class ViewFs extends AbstractFileSystem {
checkPathIsSlash(path);
throw readOnlyMountTable("deleteSnapshot", path);
}
+
+ @Override
+ public void setStoragePolicy(Path path, String policyName)
+ throws IOException {
+ throw readOnlyMountTable("setStoragePolicy", path);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a120187/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
index 374bb2e..46f24fc 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
@@ -34,6 +34,7 @@ import org.junit.Test;
import java.io.IOException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
+import java.util.Collection;
import java.util.EnumSet;
import java.util.Iterator;
import java.util.List;
@@ -205,6 +206,12 @@ public class TestHarFileSystem {
public AclStatus getAclStatus(Path path) throws IOException;
public void access(Path path, FsAction mode) throws IOException;
+
+ public void setStoragePolicy(Path src, String policyName)
+ throws IOException;
+
+ public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
+ throws IOException;
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a120187/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/BlockStoragePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/BlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/BlockStoragePolicy.java
index 9ecf6e8..2624960 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/BlockStoragePolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/BlockStoragePolicy.java
@@ -25,6 +25,7 @@ import java.util.List;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.BlockStoragePolicySpi;
import org.apache.hadoop.fs.StorageType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,7 +35,7 @@ import org.slf4j.LoggerFactory;
* for the replicas of a block.
*/
@InterfaceAudience.Private
-public class BlockStoragePolicy {
+public class BlockStoragePolicy implements BlockStoragePolicySpi {
public static final Logger LOG = LoggerFactory.getLogger(BlockStoragePolicy
.class);
@@ -239,18 +240,22 @@ public class BlockStoragePolicy {
return id;
}
+ @Override
public String getName() {
return name;
}
+ @Override
public StorageType[] getStorageTypes() {
return this.storageTypes;
}
+ @Override
public StorageType[] getCreationFallbacks() {
return this.creationFallbacks;
}
+ @Override
public StorageType[] getReplicationFallbacks() {
return this.replicationFallbacks;
}
@@ -265,6 +270,7 @@ public class BlockStoragePolicy {
return null;
}
+ @Override
public boolean isCopyOnCreateFile() {
return copyOnCreateFile;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a120187/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8d0c5b6..4270a9c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -567,6 +567,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8157. Writes to RAM DISK reserve locked memory for block files.
(Arpit Agarwal)
+ HDFS-8345. Storage policy APIs must be exposed via the FileSystem
+ interface. (Arpit Agarwal)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a120187/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
index b776849..3f78b31 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
@@ -22,6 +22,8 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
@@ -465,6 +467,17 @@ public class Hdfs extends AbstractFileSystem {
dfs.checkAccess(getUriPath(path), mode);
}
+ @Override
+ public void setStoragePolicy(Path path, String policyName) throws IOException {
+ dfs.setStoragePolicy(getUriPath(path), policyName);
+ }
+
+ @Override
+ public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
+ throws IOException {
+ return Arrays.asList(dfs.getStoragePolicies());
+ }
+
/**
* Renew an existing delegation token.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a120187/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 4ca6d57..902636c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -23,6 +23,8 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
@@ -32,6 +34,7 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.BlockStorageLocation;
+import org.apache.hadoop.fs.BlockStoragePolicySpi;
import org.apache.hadoop.fs.CacheFlag;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
@@ -532,6 +535,7 @@ public class DistributedFileSystem extends FileSystem {
* @param src The source path referring to either a directory or a file.
* @param policyName The name of the storage policy.
*/
+ @Override
public void setStoragePolicy(final Path src, final String policyName)
throws IOException {
statistics.incrementWriteOps(1);
@@ -546,19 +550,24 @@ public class DistributedFileSystem extends FileSystem {
@Override
public Void next(final FileSystem fs, final Path p)
throws IOException {
- if (fs instanceof DistributedFileSystem) {
- ((DistributedFileSystem) fs).setStoragePolicy(p, policyName);
- return null;
- } else {
- throw new UnsupportedOperationException(
- "Cannot perform setStoragePolicy on a non-DistributedFileSystem: "
- + src + " -> " + p);
- }
+ fs.setStoragePolicy(p, policyName);
+ return null;
}
}.resolve(this, absF);
}
- /** Get all the existing storage policies */
+ @Override
+ public Collection<BlockStoragePolicy> getAllStoragePolicies()
+ throws IOException {
+ return Arrays.asList(dfs.getStoragePolicies());
+ }
+
+ /**
+ * Deprecated. Prefer {@link FileSystem#getAllStoragePolicies()}
+ * @return
+ * @throws IOException
+ */
+ @Deprecated
public BlockStoragePolicy[] getStoragePolicies() throws IOException {
statistics.incrementReadOps(1);
return dfs.getStoragePolicies();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a120187/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index 0710f3e..8715ce4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -27,6 +27,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.BlockStoragePolicySpi;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.HdfsConfiguration;
@@ -152,8 +153,8 @@ public class Mover {
}
private void initStoragePolicies() throws IOException {
- BlockStoragePolicy[] policies = dispatcher.getDistributedFileSystem()
- .getStoragePolicies();
+ Collection<BlockStoragePolicy> policies =
+ dispatcher.getDistributedFileSystem().getAllStoragePolicies();
for (BlockStoragePolicy policy : policies) {
this.blockStoragePolicies[policy.getId()] = policy;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a120187/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
index e3bdffa..98c8a6b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.util.Tool;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
@@ -97,7 +98,7 @@ public class StoragePolicyAdmin extends Configured implements Tool {
public int run(Configuration conf, List<String> args) throws IOException {
final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
try {
- BlockStoragePolicy[] policies = dfs.getStoragePolicies();
+ Collection<BlockStoragePolicy> policies = dfs.getAllStoragePolicies();
System.out.println("Block Storage Policies:");
for (BlockStoragePolicy policy : policies) {
if (policy != null) {
@@ -155,7 +156,7 @@ public class StoragePolicyAdmin extends Configured implements Tool {
System.out.println("The storage policy of " + path + " is unspecified");
return 0;
}
- BlockStoragePolicy[] policies = dfs.getStoragePolicies();
+ Collection<BlockStoragePolicy> policies = dfs.getAllStoragePolicies();
for (BlockStoragePolicy p : policies) {
if (p.getId() == storagePolicyId) {
System.out.println("The storage policy of " + path + ":\n" + p);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6a120187/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index 5e3b55f..ea69f97 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -25,7 +25,9 @@ import java.io.IOException;
import java.util.*;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockStoragePolicySpi;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -1150,30 +1152,6 @@ public class TestBlockStoragePolicy {
Assert.assertEquals(3, targets.length);
}
- /**
- * Test getting all the storage policies from the namenode
- */
- @Test
- public void testGetAllStoragePolicies() throws Exception {
- final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
- .numDataNodes(0).build();
- cluster.waitActive();
- final DistributedFileSystem fs = cluster.getFileSystem();
- try {
- BlockStoragePolicy[] policies = fs.getStoragePolicies();
- Assert.assertEquals(6, policies.length);
- Assert.assertEquals(POLICY_SUITE.getPolicy(COLD).toString(),
- policies[0].toString());
- Assert.assertEquals(POLICY_SUITE.getPolicy(WARM).toString(),
- policies[1].toString());
- Assert.assertEquals(POLICY_SUITE.getPolicy(HOT).toString(),
- policies[2].toString());
- } finally {
- IOUtils.cleanup(null, fs);
- cluster.shutdown();
- }
- }
-
@Test
public void testGetFileStoragePolicyAfterRestartNN() throws Exception {
//HDFS8219
@@ -1217,4 +1195,42 @@ public class TestBlockStoragePolicy {
cluster.shutdown();
}
}
+
+ /**
+ * Verify that {@link FileSystem#getAllStoragePolicies} returns all
+ * known storage policies for DFS.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testGetAllStoragePoliciesFromFs() throws IOException {
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(REPLICATION)
+ .storageTypes(
+ new StorageType[] {StorageType.DISK, StorageType.ARCHIVE})
+ .build();
+ try {
+ cluster.waitActive();
+
+ // Get policies via {@link FileSystem#getAllStoragePolicies}
+ Set<String> policyNamesSet1 = new HashSet<>();
+ for (BlockStoragePolicySpi policy :
+ cluster.getFileSystem().getAllStoragePolicies()) {
+ policyNamesSet1.add(policy.getName());
+ }
+
+ // Get policies from the default BlockStoragePolicySuite.
+ BlockStoragePolicySuite suite = BlockStoragePolicySuite.createDefaultSuite();
+ Set<String> policyNamesSet2 = new HashSet<>();
+ for (BlockStoragePolicy policy : suite.getAllPolicies()) {
+ policyNamesSet2.add(policy.getName());
+ }
+
+ // Ensure that we got the same set of policies in both cases.
+ Assert.assertTrue(Sets.difference(policyNamesSet1, policyNamesSet2).isEmpty());
+ Assert.assertTrue(Sets.difference(policyNamesSet2, policyNamesSet1).isEmpty());
+ } finally {
+ cluster.shutdown();
+ }
+ }
}
[31/50] [abbrv] hadoop git commit: HADOOP-11963. Metrics
documentation for FSNamesystem misspells PendingDataNodeMessageCount.
Contributed by Anu Engineer.
Posted by zj...@apache.org.
HADOOP-11963. Metrics documentation for FSNamesystem misspells PendingDataNodeMessageCount. Contributed by Anu Engineer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d096beb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d096beb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d096beb
Branch: refs/heads/YARN-2928
Commit: 8d096bebeb0ac52ee6358a43f94cd49d0bfbc60f
Parents: b31e61a
Author: cnauroth <cn...@apache.org>
Authored: Tue May 19 11:50:27 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:52:10 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d096beb/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e4537a3..4621f80 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -716,6 +716,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11581. Multithreaded correctness Warnings
#org.apache.hadoop.fs.shell.Ls (Brahma Reddy Battula via aw)
+ HADOOP-11963. Metrics documentation for FSNamesystem misspells
+ PendingDataNodeMessageCount. (Anu Engineer via cnauroth)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d096beb/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index 4a10a00..ca89745 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -225,7 +225,7 @@ Each metrics record contains tags such as HAState and Hostname as additional inf
| `PendingDeletionBlocks` | Current number of blocks pending deletion |
| `ExcessBlocks` | Current number of excess blocks |
| `PostponedMisreplicatedBlocks` | (HA-only) Current number of blocks postponed to replicate |
-| `PendingDataNodeMessageCourt` | (HA-only) Current number of pending block-related messages for later processing in the standby NameNode |
+| `PendingDataNodeMessageCount` | (HA-only) Current number of pending block-related messages for later processing in the standby NameNode |
| `MillisSinceLastLoadedEdits` | (HA-only) Time in milliseconds since the last time standby NameNode load edit log. In active NameNode, set to 0 |
| `BlockCapacity` | Current number of block capacity |
| `StaleDataNodes` | Current number of DataNodes marked stale due to delayed heartbeat |
[37/50] [abbrv] hadoop git commit: HADOOP-11995. Make jetty version
configurable from the maven command line. Contributed by Sriharsha Devineni.
Posted by zj...@apache.org.
HADOOP-11995. Make jetty version configurable from the maven command line. Contributed by Sriharsha Devineni.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/440ba1e5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/440ba1e5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/440ba1e5
Branch: refs/heads/YARN-2928
Commit: 440ba1e564e7f40c2be945e01f551435e082a958
Parents: 31274c1
Author: Haohui Mai <wh...@apache.org>
Authored: Tue May 19 18:28:20 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 12:42:56 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
hadoop-project/pom.xml | 8 ++++----
2 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/440ba1e5/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 3e7cb39..b0b8fb7 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -587,6 +587,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11103. Clean up RemoteException (Sean Busbey via vinayakumarb)
+ HADOOP-11995. Make jetty version configurable from the maven command line.
+ (Sriharsha Devineni via wheat9)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/440ba1e5/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 2c53318..0889241 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -32,7 +32,7 @@
<properties>
<failIfNoTests>false</failIfNoTests>
<maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile>
-
+ <jetty.version>6.1.26</jetty.version>
<test.exclude>_</test.exclude>
<test.exclude.pattern>_</test.exclude.pattern>
@@ -483,7 +483,7 @@
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
- <version>6.1.26</version>
+ <version>${jetty.version}</version>
<exclusions>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
@@ -494,7 +494,7 @@
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
- <version>6.1.26</version>
+ <version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
@@ -613,7 +613,7 @@
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-servlet-tester</artifactId>
- <version>6.1.26</version>
+ <version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
[42/50] [abbrv] hadoop git commit: YARN-3654. ContainerLogsPage web
UI should not have meta-refresh. Contributed by Xuan Gong
Posted by zj...@apache.org.
YARN-3654. ContainerLogsPage web UI should not have meta-refresh. Contributed by Xuan Gong
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/458de345
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/458de345
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/458de345
Branch: refs/heads/YARN-2928
Commit: 458de345a7e9f873da215ebc1ba0d47e9c261ddd
Parents: 64074a3
Author: Jian He <ji...@apache.org>
Authored: Wed May 20 17:20:21 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 12:42:57 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../nodemanager/webapp/ContainerLogsPage.java | 3 -
.../server/nodemanager/webapp/NMController.java | 40 +------
.../nodemanager/webapp/NMWebAppFilter.java | 118 +++++++++++++++++++
.../server/nodemanager/webapp/WebServer.java | 6 +
5 files changed, 128 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/458de345/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index fa796ac..863211c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -507,6 +507,9 @@ Release 2.8.0 - UNRELEASED
YARN-2821. Fixed a problem that DistributedShell AM may hang if restarted.
(Varun Vasudev via jianhe)
+ YARN-3654. ContainerLogsPage web UI should not have meta-refresh. (Xuan Gong
+ via jianhe)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/458de345/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
index 48e0c87..2fd6b2c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
@@ -59,9 +59,6 @@ public class ContainerLogsPage extends NMView {
if (redirectUrl.equals("false")) {
set(TITLE, join("Failed redirect for ", $(CONTAINER_ID)));
//Error getting redirect url. Fall through.
- } else {
- set(TITLE, join("Redirecting to log server for ", $(CONTAINER_ID)));
- html.meta_http("refresh", "1; url=" + redirectUrl);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/458de345/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java
index 097532f..5be5b35 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java
@@ -20,29 +20,16 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp;
import static org.apache.hadoop.yarn.util.StringHelper.join;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.Controller;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import com.google.inject.Inject;
public class NMController extends Controller implements YarnWebParams {
-
- private Context nmContext;
- private Configuration nmConf;
@Inject
- public NMController(Configuration nmConf, RequestContext requestContext,
- Context nmContext) {
+ public NMController(RequestContext requestContext) {
super(requestContext);
- this.nmContext = nmContext;
- this.nmConf = nmConf;
}
@Override
@@ -80,31 +67,6 @@ public class NMController extends Controller implements YarnWebParams {
}
public void logs() {
- String containerIdStr = $(CONTAINER_ID);
- ContainerId containerId = null;
- try {
- containerId = ConverterUtils.toContainerId(containerIdStr);
- } catch (IllegalArgumentException e) {
- render(ContainerLogsPage.class);
- return;
- }
- ApplicationId appId =
- containerId.getApplicationAttemptId().getApplicationId();
- Application app = nmContext.getApplications().get(appId);
- if (app == null
- && nmConf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
- YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED)) {
- String logServerUrl = nmConf.get(YarnConfiguration.YARN_LOG_SERVER_URL);
- String redirectUrl = null;
- if (logServerUrl == null || logServerUrl.isEmpty()) {
- redirectUrl = "false";
- } else {
- redirectUrl =
- url(logServerUrl, nmContext.getNodeId().toString(), containerIdStr,
- containerIdStr, $(APP_OWNER));
- }
- set(ContainerLogsPage.REDIRECT_URL, redirectUrl);
- }
render(ContainerLogsPage.class);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/458de345/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebAppFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebAppFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebAppFilter.java
new file mode 100644
index 0000000..63fe6ea
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebAppFilter.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.webapp;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HtmlQuoting;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
+import org.apache.hadoop.yarn.webapp.Controller.RequestContext;
+import com.google.inject.Injector;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+
+@Singleton
+public class NMWebAppFilter extends GuiceContainer{
+
+ private Injector injector;
+ private Context nmContext;
+
+ private static final long serialVersionUID = 1L;
+
+ @Inject
+ public NMWebAppFilter(Injector injector, Context nmContext) {
+ super(injector);
+ this.injector = injector;
+ this.nmContext = nmContext;
+ }
+
+ @Override
+ public void doFilter(HttpServletRequest request,
+ HttpServletResponse response, FilterChain chain) throws IOException,
+ ServletException {
+ String uri = HtmlQuoting.quoteHtmlChars(request.getRequestURI());
+ String redirectPath = containerLogPageRedirectPath(uri);
+ if (redirectPath != null) {
+ String redirectMsg =
+ "Redirecting to log server" + " : " + redirectPath;
+ PrintWriter out = response.getWriter();
+ out.println(redirectMsg);
+ response.setHeader("Location", redirectPath);
+ response.setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT);
+ return;
+ }
+ super.doFilter(request, response, chain);
+ }
+
+ private String containerLogPageRedirectPath(String uri) {
+ String redirectPath = null;
+ if (!uri.contains("/ws/v1/node") && uri.contains("/containerlogs")) {
+ String[] parts = uri.split("/");
+ String containerIdStr = parts[3];
+ String appOwner = parts[4];
+ if (containerIdStr != null && !containerIdStr.isEmpty()) {
+ ContainerId containerId = null;
+ try {
+ containerId = ContainerId.fromString(containerIdStr);
+ } catch (IllegalArgumentException ex) {
+ return redirectPath;
+ }
+ ApplicationId appId =
+ containerId.getApplicationAttemptId().getApplicationId();
+ Application app = nmContext.getApplications().get(appId);
+ Configuration nmConf = nmContext.getLocalDirsHandler().getConfig();
+ if (app == null
+ && nmConf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
+ YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED)) {
+ String logServerUrl =
+ nmConf.get(YarnConfiguration.YARN_LOG_SERVER_URL);
+ if (logServerUrl != null && !logServerUrl.isEmpty()) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(logServerUrl);
+ sb.append("/");
+ sb.append(nmContext.getNodeId().toString());
+ sb.append("/");
+ sb.append(containerIdStr);
+ sb.append("/");
+ sb.append(containerIdStr);
+ sb.append("/");
+ sb.append(appOwner);
+ redirectPath = sb.toString();
+ } else {
+ injector.getInstance(RequestContext.class).set(
+ ContainerLogsPage.REDIRECT_URL, "false");
+ }
+ }
+ }
+ }
+ return redirectPath;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/458de345/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
index 2b6b549..0e0c1e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
@@ -35,6 +35,8 @@ import org.apache.hadoop.yarn.webapp.WebApps;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+
public class WebServer extends AbstractService {
private static final Log LOG = LogFactory.getLog(WebServer.class);
@@ -128,5 +130,9 @@ public class WebServer extends AbstractService {
route("/errors-and-warnings", NMController.class, "errorsAndWarnings");
}
+ @Override
+ protected Class<? extends GuiceContainer> getWebAppFilterClass() {
+ return NMWebAppFilter.class;
+ }
}
}
[50/50] [abbrv] hadoop git commit: HDFS-8421. Move startFile() and
related functions into FSDirWriteFileOp. Contributed by Haohui Mai.
Posted by zj...@apache.org.
HDFS-8421. Move startFile() and related functions into FSDirWriteFileOp. Contributed by Haohui Mai.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e19566a6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e19566a6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e19566a6
Branch: refs/heads/YARN-2928
Commit: e19566a6603754c86e3ab955aa7f58fb7c4c4d29
Parents: 369d7c4
Author: Haohui Mai <wh...@apache.org>
Authored: Thu May 21 08:05:10 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 12:42:59 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/namenode/FSDirWriteFileOp.java | 324 ++++++++++++++++++-
.../hdfs/server/namenode/FSDirectory.java | 91 ------
.../hdfs/server/namenode/FSEditLogLoader.java | 15 +-
.../hdfs/server/namenode/FSNamesystem.java | 280 +++-------------
5 files changed, 371 insertions(+), 342 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e19566a6/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 77d7369..9cfad7d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -577,6 +577,9 @@ Release 2.8.0 - UNRELEASED
HDFS-4383. Document the lease limits. (Arshad Mohammad via aajisaka)
+ HDFS-8421. Move startFile() and related functions into FSDirWriteFileOp.
+ (wheat9)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e19566a6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 1ff0899..307bd59 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -18,11 +18,27 @@
package org.apache.hadoop.hdfs.server.namenode;
import com.google.common.base.Preconditions;
+import org.apache.commons.io.Charsets;
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.crypto.CipherSuite;
+import org.apache.hadoop.crypto.CryptoProtocolVersion;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileEncryptionInfo;
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
@@ -34,15 +50,22 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.net.NodeBase;
+import org.apache.hadoop.util.ChunkedArrayList;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
+import static org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.CURRENT_STATE_ID;
+import static org.apache.hadoop.util.Time.now;
+
class FSDirWriteFileOp {
private FSDirWriteFileOp() {}
static boolean unprotectedRemoveBlock(
@@ -278,6 +301,210 @@ class FSDirWriteFileOp {
}
/**
+ * Create a new file or overwrite an existing file<br>
+ *
+ * Once the file is create the client then allocates a new block with the next
+ * call using {@link ClientProtocol#addBlock}.
+ * <p>
+ * For description of parameters and exceptions thrown see
+ * {@link ClientProtocol#create}
+ */
+ static HdfsFileStatus startFile(
+ FSNamesystem fsn, FSPermissionChecker pc, String src,
+ PermissionStatus permissions, String holder, String clientMachine,
+ EnumSet<CreateFlag> flag, boolean createParent,
+ short replication, long blockSize,
+ EncryptionKeyInfo ezInfo, INode.BlocksMapUpdateInfo toRemoveBlocks,
+ boolean logRetryEntry)
+ throws IOException {
+ assert fsn.hasWriteLock();
+
+ boolean create = flag.contains(CreateFlag.CREATE);
+ boolean overwrite = flag.contains(CreateFlag.OVERWRITE);
+ boolean isLazyPersist = flag.contains(CreateFlag.LAZY_PERSIST);
+
+ CipherSuite suite = null;
+ CryptoProtocolVersion version = null;
+ KeyProviderCryptoExtension.EncryptedKeyVersion edek = null;
+
+ if (ezInfo != null) {
+ edek = ezInfo.edek;
+ suite = ezInfo.suite;
+ version = ezInfo.protocolVersion;
+ }
+
+ boolean isRawPath = FSDirectory.isReservedRawName(src);
+ FSDirectory fsd = fsn.getFSDirectory();
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ src = fsd.resolvePath(pc, src, pathComponents);
+ INodesInPath iip = fsd.getINodesInPath4Write(src);
+
+ // Verify that the destination does not exist as a directory already.
+ final INode inode = iip.getLastINode();
+ if (inode != null && inode.isDirectory()) {
+ throw new FileAlreadyExistsException(src +
+ " already exists as a directory");
+ }
+
+ final INodeFile myFile = INodeFile.valueOf(inode, src, true);
+ if (fsd.isPermissionEnabled()) {
+ if (overwrite && myFile != null) {
+ fsd.checkPathAccess(pc, iip, FsAction.WRITE);
+ }
+ /*
+ * To overwrite existing file, need to check 'w' permission
+ * of parent (equals to ancestor in this case)
+ */
+ fsd.checkAncestorAccess(pc, iip, FsAction.WRITE);
+ }
+
+ if (!createParent) {
+ fsd.verifyParentDir(iip, src);
+ }
+
+ if (myFile == null && !create) {
+ throw new FileNotFoundException("Can't overwrite non-existent " +
+ src + " for client " + clientMachine);
+ }
+
+ FileEncryptionInfo feInfo = null;
+
+ final EncryptionZone zone = fsd.getEZForPath(iip);
+ if (zone != null) {
+ // The path is now within an EZ, but we're missing encryption parameters
+ if (suite == null || edek == null) {
+ throw new RetryStartFileException();
+ }
+ // Path is within an EZ and we have provided encryption parameters.
+ // Make sure that the generated EDEK matches the settings of the EZ.
+ final String ezKeyName = zone.getKeyName();
+ if (!ezKeyName.equals(edek.getEncryptionKeyName())) {
+ throw new RetryStartFileException();
+ }
+ feInfo = new FileEncryptionInfo(suite, version,
+ edek.getEncryptedKeyVersion().getMaterial(),
+ edek.getEncryptedKeyIv(),
+ ezKeyName, edek.getEncryptionKeyVersionName());
+ }
+
+ if (myFile != null) {
+ if (overwrite) {
+ List<INode> toRemoveINodes = new ChunkedArrayList<>();
+ List<Long> toRemoveUCFiles = new ChunkedArrayList<>();
+ long ret = FSDirDeleteOp.delete(fsd, iip, toRemoveBlocks,
+ toRemoveINodes, toRemoveUCFiles, now());
+ if (ret >= 0) {
+ iip = INodesInPath.replace(iip, iip.length() - 1, null);
+ FSDirDeleteOp.incrDeletedFileCount(ret);
+ fsn.removeLeasesAndINodes(toRemoveUCFiles, toRemoveINodes, true);
+ }
+ } else {
+ // If lease soft limit time is expired, recover the lease
+ fsn.recoverLeaseInternal(FSNamesystem.RecoverLeaseOp.CREATE_FILE, iip,
+ src, holder, clientMachine, false);
+ throw new FileAlreadyExistsException(src + " for client " +
+ clientMachine + " already exists");
+ }
+ }
+ fsn.checkFsObjectLimit();
+ INodeFile newNode = null;
+ Map.Entry<INodesInPath, String> parent = FSDirMkdirOp
+ .createAncestorDirectories(fsd, iip, permissions);
+ if (parent != null) {
+ iip = addFile(fsd, parent.getKey(), parent.getValue(), permissions,
+ replication, blockSize, holder, clientMachine);
+ newNode = iip != null ? iip.getLastINode().asFile() : null;
+ }
+ if (newNode == null) {
+ throw new IOException("Unable to add " + src + " to namespace");
+ }
+ fsn.leaseManager.addLease(
+ newNode.getFileUnderConstructionFeature().getClientName(),
+ newNode.getId());
+ if (feInfo != null) {
+ fsd.setFileEncryptionInfo(src, feInfo);
+ newNode = fsd.getInode(newNode.getId()).asFile();
+ }
+ setNewINodeStoragePolicy(fsn.getBlockManager(), newNode, iip,
+ isLazyPersist);
+ fsd.getEditLog().logOpenFile(src, newNode, overwrite, logRetryEntry);
+ if (NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("DIR* NameSystem.startFile: added " +
+ src + " inode " + newNode.getId() + " " + holder);
+ }
+ return FSDirStatAndListingOp.getFileInfo(fsd, src, false, isRawPath, true);
+ }
+
+ static EncryptionKeyInfo getEncryptionKeyInfo(FSNamesystem fsn,
+ FSPermissionChecker pc, String src,
+ CryptoProtocolVersion[] supportedVersions)
+ throws IOException {
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ FSDirectory fsd = fsn.getFSDirectory();
+ src = fsd.resolvePath(pc, src, pathComponents);
+ INodesInPath iip = fsd.getINodesInPath4Write(src);
+ // Nothing to do if the path is not within an EZ
+ final EncryptionZone zone = fsd.getEZForPath(iip);
+ if (zone == null) {
+ return null;
+ }
+ CryptoProtocolVersion protocolVersion = fsn.chooseProtocolVersion(
+ zone, supportedVersions);
+ CipherSuite suite = zone.getSuite();
+ String ezKeyName = zone.getKeyName();
+
+ Preconditions.checkNotNull(protocolVersion);
+ Preconditions.checkNotNull(suite);
+ Preconditions.checkArgument(!suite.equals(CipherSuite.UNKNOWN),
+ "Chose an UNKNOWN CipherSuite!");
+ Preconditions.checkNotNull(ezKeyName);
+ return new EncryptionKeyInfo(protocolVersion, suite, ezKeyName);
+ }
+
+ static INodeFile addFileForEditLog(
+ FSDirectory fsd, long id, INodesInPath existing, byte[] localName,
+ PermissionStatus permissions, List<AclEntry> aclEntries,
+ List<XAttr> xAttrs, short replication, long modificationTime, long atime,
+ long preferredBlockSize, boolean underConstruction, String clientName,
+ String clientMachine, byte storagePolicyId) {
+ final INodeFile newNode;
+ assert fsd.hasWriteLock();
+ if (underConstruction) {
+ newNode = newINodeFile(id, permissions, modificationTime,
+ modificationTime, replication,
+ preferredBlockSize,
+ storagePolicyId);
+ newNode.toUnderConstruction(clientName, clientMachine);
+ } else {
+ newNode = newINodeFile(id, permissions, modificationTime,
+ atime, replication,
+ preferredBlockSize,
+ storagePolicyId);
+ }
+
+ newNode.setLocalName(localName);
+ try {
+ INodesInPath iip = fsd.addINode(existing, newNode);
+ if (iip != null) {
+ if (aclEntries != null) {
+ AclStorage.updateINodeAcl(newNode, aclEntries, CURRENT_STATE_ID);
+ }
+ if (xAttrs != null) {
+ XAttrStorage.updateINodeXAttrs(newNode, xAttrs, CURRENT_STATE_ID);
+ }
+ return newNode;
+ }
+ } catch (IOException e) {
+ if(NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug(
+ "DIR* FSDirectory.unprotectedAddFile: exception when add "
+ + existing.getPath() + " to the file system", e);
+ }
+ }
+ return null;
+ }
+
+ /**
* Add a block to the file. Returns a reference to the added block.
*/
private static BlockInfoContiguous addBlock(
@@ -314,6 +541,41 @@ class FSDirWriteFileOp {
}
}
+ /**
+ * Add the given filename to the fs.
+ * @return the new INodesInPath instance that contains the new INode
+ */
+ private static INodesInPath addFile(
+ FSDirectory fsd, INodesInPath existing, String localName,
+ PermissionStatus permissions, short replication, long preferredBlockSize,
+ String clientName, String clientMachine)
+ throws IOException {
+
+ long modTime = now();
+ INodeFile newNode = newINodeFile(fsd.allocateNewInodeId(), permissions,
+ modTime, modTime, replication, preferredBlockSize);
+ newNode.setLocalName(localName.getBytes(Charsets.UTF_8));
+ newNode.toUnderConstruction(clientName, clientMachine);
+
+ INodesInPath newiip;
+ fsd.writeLock();
+ try {
+ newiip = fsd.addINode(existing, newNode);
+ } finally {
+ fsd.writeUnlock();
+ }
+ if (newiip == null) {
+ NameNode.stateChangeLog.info("DIR* addFile: failed to add " +
+ existing.getPath() + "/" + localName);
+ return null;
+ }
+
+ if(NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("DIR* addFile: " + localName + " is added");
+ }
+ return newiip;
+ }
+
private static FileState analyzeFileState(
FSNamesystem fsn, String src, long fileId, String clientName,
ExtendedBlock previous, LocatedBlock[] onRetryBlock)
@@ -345,8 +607,7 @@ class FSDirWriteFileOp {
src = iip.getPath();
}
}
- final INodeFile file = fsn.checkLease(src, clientName,
- inode, fileId);
+ final INodeFile file = fsn.checkLease(src, clientName, inode, fileId);
BlockInfoContiguous lastBlockInFile = file.getLastBlock();
if (!Block.matchingIdAndGenStamp(previousBlock, lastBlockInFile)) {
// The block that the client claims is the current last block
@@ -497,6 +758,20 @@ class FSDirWriteFileOp {
return true;
}
+ private static INodeFile newINodeFile(
+ long id, PermissionStatus permissions, long mtime, long atime,
+ short replication, long preferredBlockSize, byte storagePolicyId) {
+ return new INodeFile(id, null, permissions, mtime, atime,
+ BlockInfoContiguous.EMPTY_ARRAY, replication, preferredBlockSize,
+ storagePolicyId);
+ }
+
+ private static INodeFile newINodeFile(long id, PermissionStatus permissions,
+ long mtime, long atime, short replication, long preferredBlockSize) {
+ return newINodeFile(id, permissions, mtime, atime, replication,
+ preferredBlockSize, (byte)0);
+ }
+
/**
* Persist the new block (the last block of the given file).
*/
@@ -533,6 +808,36 @@ class FSDirWriteFileOp {
DatanodeStorageInfo.incrementBlocksScheduled(targets);
}
+ private static void setNewINodeStoragePolicy(BlockManager bm, INodeFile
+ inode, INodesInPath iip, boolean isLazyPersist)
+ throws IOException {
+
+ if (isLazyPersist) {
+ BlockStoragePolicy lpPolicy =
+ bm.getStoragePolicy("LAZY_PERSIST");
+
+ // Set LAZY_PERSIST storage policy if the flag was passed to
+ // CreateFile.
+ if (lpPolicy == null) {
+ throw new HadoopIllegalArgumentException(
+ "The LAZY_PERSIST storage policy has been disabled " +
+ "by the administrator.");
+ }
+ inode.setStoragePolicyID(lpPolicy.getId(),
+ iip.getLatestSnapshotId());
+ } else {
+ BlockStoragePolicy effectivePolicy =
+ bm.getStoragePolicy(inode.getStoragePolicyID());
+
+ if (effectivePolicy != null &&
+ effectivePolicy.isCopyOnCreateFile()) {
+ // Copy effective policy from ancestor directory to current file.
+ inode.setStoragePolicyID(effectivePolicy.getId(),
+ iip.getLatestSnapshotId());
+ }
+ }
+ }
+
private static class FileState {
final INodeFile inode;
final String path;
@@ -560,4 +865,19 @@ class FSDirWriteFileOp {
this.clientMachine = clientMachine;
}
}
+
+ static class EncryptionKeyInfo {
+ final CryptoProtocolVersion protocolVersion;
+ final CipherSuite suite;
+ final String ezKeyName;
+ KeyProviderCryptoExtension.EncryptedKeyVersion edek;
+
+ EncryptionKeyInfo(
+ CryptoProtocolVersion protocolVersion, CipherSuite suite,
+ String ezKeyName) {
+ this.protocolVersion = protocolVersion;
+ this.suite = suite;
+ this.ezKeyName = ezKeyName;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e19566a6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index c2ed956..8fdd2d7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -21,13 +21,11 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.commons.io.Charsets;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.CipherSuite;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
@@ -42,7 +40,6 @@ import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.XAttrHelper;
-import org.apache.hadoop.hdfs.protocol.AclException;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
@@ -86,7 +83,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KE
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_ENCRYPTION_ZONE;
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_FILE_ENCRYPTION_INFO;
import static org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.CURRENT_STATE_ID;
-import static org.apache.hadoop.util.Time.now;
/**
* Both FSDirectory and FSNamesystem manage the state of the namespace.
@@ -388,93 +384,6 @@ public class FSDirectory implements Closeable {
skipQuotaCheck = true;
}
- private static INodeFile newINodeFile(long id, PermissionStatus permissions,
- long mtime, long atime, short replication, long preferredBlockSize) {
- return newINodeFile(id, permissions, mtime, atime, replication,
- preferredBlockSize, (byte)0);
- }
-
- private static INodeFile newINodeFile(long id, PermissionStatus permissions,
- long mtime, long atime, short replication, long preferredBlockSize,
- byte storagePolicyId) {
- return new INodeFile(id, null, permissions, mtime, atime,
- BlockInfoContiguous.EMPTY_ARRAY, replication, preferredBlockSize,
- storagePolicyId);
- }
-
- /**
- * Add the given filename to the fs.
- * @return the new INodesInPath instance that contains the new INode
- */
- INodesInPath addFile(INodesInPath existing, String localName, PermissionStatus
- permissions, short replication, long preferredBlockSize,
- String clientName, String clientMachine)
- throws FileAlreadyExistsException, QuotaExceededException,
- UnresolvedLinkException, SnapshotAccessControlException, AclException {
-
- long modTime = now();
- INodeFile newNode = newINodeFile(allocateNewInodeId(), permissions, modTime,
- modTime, replication, preferredBlockSize);
- newNode.setLocalName(localName.getBytes(Charsets.UTF_8));
- newNode.toUnderConstruction(clientName, clientMachine);
-
- INodesInPath newiip;
- writeLock();
- try {
- newiip = addINode(existing, newNode);
- } finally {
- writeUnlock();
- }
- if (newiip == null) {
- NameNode.stateChangeLog.info("DIR* addFile: failed to add " +
- existing.getPath() + "/" + localName);
- return null;
- }
-
- if(NameNode.stateChangeLog.isDebugEnabled()) {
- NameNode.stateChangeLog.debug("DIR* addFile: " + localName + " is added");
- }
- return newiip;
- }
-
- INodeFile addFileForEditLog(long id, INodesInPath existing, byte[] localName,
- PermissionStatus permissions, List<AclEntry> aclEntries,
- List<XAttr> xAttrs, short replication, long modificationTime, long atime,
- long preferredBlockSize, boolean underConstruction, String clientName,
- String clientMachine, byte storagePolicyId) {
- final INodeFile newNode;
- assert hasWriteLock();
- if (underConstruction) {
- newNode = newINodeFile(id, permissions, modificationTime,
- modificationTime, replication, preferredBlockSize, storagePolicyId);
- newNode.toUnderConstruction(clientName, clientMachine);
- } else {
- newNode = newINodeFile(id, permissions, modificationTime, atime,
- replication, preferredBlockSize, storagePolicyId);
- }
-
- newNode.setLocalName(localName);
- try {
- INodesInPath iip = addINode(existing, newNode);
- if (iip != null) {
- if (aclEntries != null) {
- AclStorage.updateINodeAcl(newNode, aclEntries, CURRENT_STATE_ID);
- }
- if (xAttrs != null) {
- XAttrStorage.updateINodeXAttrs(newNode, xAttrs, CURRENT_STATE_ID);
- }
- return newNode;
- }
- } catch (IOException e) {
- if(NameNode.stateChangeLog.isDebugEnabled()) {
- NameNode.stateChangeLog.debug(
- "DIR* FSDirectory.unprotectedAddFile: exception when add "
- + existing.getPath() + " to the file system", e);
- }
- }
- return null;
- }
-
/**
* This is a wrapper for resolvePath(). If the path passed
* is prefixed with /.reserved/raw, then it checks to ensure that the caller
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e19566a6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index dec1298..476ff36 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -364,15 +364,12 @@ public class FSEditLogLoader {
// add to the file tree
inodeId = getAndUpdateLastInodeId(addCloseOp.inodeId, logVersion, lastInodeId);
- newFile = fsDir.addFileForEditLog(inodeId, iip.getExistingINodes(),
- iip.getLastLocalName(),
- addCloseOp.permissions,
- addCloseOp.aclEntries,
- addCloseOp.xAttrs, replication,
- addCloseOp.mtime, addCloseOp.atime,
- addCloseOp.blockSize, true,
- addCloseOp.clientName,
- addCloseOp.clientMachine,
+ newFile = FSDirWriteFileOp.addFileForEditLog(fsDir, inodeId,
+ iip.getExistingINodes(), iip.getLastLocalName(),
+ addCloseOp.permissions, addCloseOp.aclEntries,
+ addCloseOp.xAttrs, replication, addCloseOp.mtime,
+ addCloseOp.atime, addCloseOp.blockSize, true,
+ addCloseOp.clientName, addCloseOp.clientMachine,
addCloseOp.storagePolicyId);
iip = INodesInPath.replace(iip, iip.length() - 1, newFile);
fsNamesys.leaseManager.addLease(addCloseOp.clientName, newFile.getId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e19566a6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 7e5b981..bfd6eba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -151,7 +151,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.Options;
-import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.XAttr;
@@ -275,7 +274,6 @@ import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.token.delegation.DelegationKey;
-import org.apache.hadoop.util.ChunkedArrayList;
import org.apache.hadoop.util.Daemon;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.ReflectionUtils;
@@ -2279,8 +2277,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* @return chosen protocol version
* @throws IOException
*/
- private CryptoProtocolVersion chooseProtocolVersion(EncryptionZone zone,
- CryptoProtocolVersion[] supportedVersions)
+ CryptoProtocolVersion chooseProtocolVersion(
+ EncryptionZone zone, CryptoProtocolVersion[] supportedVersions)
throws UnknownCryptoProtocolVersionException, UnresolvedLinkException,
SnapshotAccessControlException {
Preconditions.checkNotNull(zone);
@@ -2342,11 +2340,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
String holder, String clientMachine, EnumSet<CreateFlag> flag,
boolean createParent, short replication, long blockSize,
CryptoProtocolVersion[] supportedVersions, boolean logRetryCache)
- throws AccessControlException, SafeModeException,
- FileAlreadyExistsException, UnresolvedLinkException,
- FileNotFoundException, ParentNotDirectoryException, IOException {
+ throws IOException {
- HdfsFileStatus status = null;
+ HdfsFileStatus status;
try {
status = startFileInt(src, permissions, holder, clientMachine, flag,
createParent, replication, blockSize, supportedVersions,
@@ -2355,54 +2351,42 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
logAuditEvent(false, "create", src);
throw e;
}
+ logAuditEvent(true, "create", src, null, status);
return status;
}
- private HdfsFileStatus startFileInt(final String srcArg,
+ private HdfsFileStatus startFileInt(final String src,
PermissionStatus permissions, String holder, String clientMachine,
EnumSet<CreateFlag> flag, boolean createParent, short replication,
long blockSize, CryptoProtocolVersion[] supportedVersions,
boolean logRetryCache)
- throws AccessControlException, SafeModeException,
- FileAlreadyExistsException, UnresolvedLinkException,
- FileNotFoundException, ParentNotDirectoryException, IOException {
- String src = srcArg;
+ throws IOException {
if (NameNode.stateChangeLog.isDebugEnabled()) {
StringBuilder builder = new StringBuilder();
- builder.append("DIR* NameSystem.startFile: src=" + src
- + ", holder=" + holder
- + ", clientMachine=" + clientMachine
- + ", createParent=" + createParent
- + ", replication=" + replication
- + ", createFlag=" + flag.toString()
- + ", blockSize=" + blockSize);
- builder.append(", supportedVersions=");
- if (supportedVersions != null) {
- builder.append(Arrays.toString(supportedVersions));
- } else {
- builder.append("null");
- }
+ builder.append("DIR* NameSystem.startFile: src=").append(src)
+ .append(", holder=").append(holder)
+ .append(", clientMachine=").append(clientMachine)
+ .append(", createParent=").append(createParent)
+ .append(", replication=").append(replication)
+ .append(", createFlag=").append(flag.toString())
+ .append(", blockSize=").append(blockSize)
+ .append(", supportedVersions=")
+ .append(supportedVersions == null ? null : Arrays.toString
+ (supportedVersions));
NameNode.stateChangeLog.debug(builder.toString());
}
if (!DFSUtil.isValidName(src)) {
throw new InvalidPathException(src);
}
blockManager.verifyReplication(src, replication, clientMachine);
-
- boolean skipSync = false;
- HdfsFileStatus stat = null;
- FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
if (blockSize < minBlockSize) {
throw new IOException("Specified block size is less than configured" +
" minimum value (" + DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY
+ "): " + blockSize + " < " + minBlockSize);
}
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
- boolean create = flag.contains(CreateFlag.CREATE);
- boolean overwrite = flag.contains(CreateFlag.OVERWRITE);
- boolean isLazyPersist = flag.contains(CreateFlag.LAZY_PERSIST);
+ FSPermissionChecker pc = getPermissionChecker();
waitForLoadingFSImage();
/**
@@ -2417,246 +2401,62 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* special RetryStartFileException to ask the DFSClient to try the create
* again later.
*/
- CryptoProtocolVersion protocolVersion = null;
- CipherSuite suite = null;
- String ezKeyName = null;
- EncryptedKeyVersion edek = null;
+ FSDirWriteFileOp.EncryptionKeyInfo ezInfo = null;
if (provider != null) {
readLock();
try {
- src = dir.resolvePath(pc, src, pathComponents);
- INodesInPath iip = dir.getINodesInPath4Write(src);
- // Nothing to do if the path is not within an EZ
- final EncryptionZone zone = dir.getEZForPath(iip);
- if (zone != null) {
- protocolVersion = chooseProtocolVersion(zone, supportedVersions);
- suite = zone.getSuite();
- ezKeyName = zone.getKeyName();
-
- Preconditions.checkNotNull(protocolVersion);
- Preconditions.checkNotNull(suite);
- Preconditions.checkArgument(!suite.equals(CipherSuite.UNKNOWN),
- "Chose an UNKNOWN CipherSuite!");
- Preconditions.checkNotNull(ezKeyName);
- }
+ checkOperation(OperationCategory.READ);
+ ezInfo = FSDirWriteFileOp
+ .getEncryptionKeyInfo(this, pc, src, supportedVersions);
} finally {
readUnlock();
}
- Preconditions.checkState(
- (suite == null && ezKeyName == null) ||
- (suite != null && ezKeyName != null),
- "Both suite and ezKeyName should both be null or not null");
-
// Generate EDEK if necessary while not holding the lock
- edek = generateEncryptedDataEncryptionKey(ezKeyName);
+ if (ezInfo != null) {
+ ezInfo.edek = generateEncryptedDataEncryptionKey(ezInfo.ezKeyName);
+ }
EncryptionFaultInjector.getInstance().startFileAfterGenerateKey();
}
- // Proceed with the create, using the computed cipher suite and
+ boolean skipSync = false;
+ HdfsFileStatus stat = null;
+
+ // Proceed with the create, using the computed cipher suite and
// generated EDEK
- BlocksMapUpdateInfo toRemoveBlocks = null;
+ BlocksMapUpdateInfo toRemoveBlocks = new BlocksMapUpdateInfo();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot create file" + src);
dir.writeLock();
try {
- src = dir.resolvePath(pc, src, pathComponents);
- final INodesInPath iip = dir.getINodesInPath4Write(src);
- toRemoveBlocks = startFileInternal(
- pc, iip, permissions, holder,
- clientMachine, create, overwrite,
- createParent, replication, blockSize,
- isLazyPersist, suite, protocolVersion, edek,
- logRetryCache);
- stat = FSDirStatAndListingOp.getFileInfo(
- dir, src, false, FSDirectory.isReservedRawName(srcArg), true);
+ stat = FSDirWriteFileOp.startFile(this, pc, src, permissions, holder,
+ clientMachine, flag, createParent,
+ replication, blockSize, ezInfo,
+ toRemoveBlocks, logRetryCache);
} finally {
dir.writeUnlock();
}
- } catch (StandbyException se) {
- skipSync = true;
- throw se;
+ } catch (IOException e) {
+ skipSync = e instanceof StandbyException;
+ throw e;
} finally {
writeUnlock();
// There might be transactions logged while trying to recover the lease.
// They need to be sync'ed even when an exception was thrown.
if (!skipSync) {
getEditLog().logSync();
- if (toRemoveBlocks != null) {
- removeBlocks(toRemoveBlocks);
- toRemoveBlocks.clear();
- }
+ removeBlocks(toRemoveBlocks);
+ toRemoveBlocks.clear();
}
}
- logAuditEvent(true, "create", srcArg, null, stat);
return stat;
}
/**
- * Create a new file or overwrite an existing file<br>
- *
- * Once the file is create the client then allocates a new block with the next
- * call using {@link ClientProtocol#addBlock}.
- * <p>
- * For description of parameters and exceptions thrown see
- * {@link ClientProtocol#create}
- */
- private BlocksMapUpdateInfo startFileInternal(FSPermissionChecker pc,
- INodesInPath iip, PermissionStatus permissions, String holder,
- String clientMachine, boolean create, boolean overwrite,
- boolean createParent, short replication, long blockSize,
- boolean isLazyPersist, CipherSuite suite, CryptoProtocolVersion version,
- EncryptedKeyVersion edek, boolean logRetryEntry)
- throws IOException {
- assert hasWriteLock();
- // Verify that the destination does not exist as a directory already.
- final INode inode = iip.getLastINode();
- final String src = iip.getPath();
- if (inode != null && inode.isDirectory()) {
- throw new FileAlreadyExistsException(src +
- " already exists as a directory");
- }
-
- final INodeFile myFile = INodeFile.valueOf(inode, src, true);
- if (isPermissionEnabled) {
- if (overwrite && myFile != null) {
- dir.checkPathAccess(pc, iip, FsAction.WRITE);
- }
- /*
- * To overwrite existing file, need to check 'w' permission
- * of parent (equals to ancestor in this case)
- */
- dir.checkAncestorAccess(pc, iip, FsAction.WRITE);
- }
- if (!createParent) {
- dir.verifyParentDir(iip, src);
- }
-
- FileEncryptionInfo feInfo = null;
-
- final EncryptionZone zone = dir.getEZForPath(iip);
- if (zone != null) {
- // The path is now within an EZ, but we're missing encryption parameters
- if (suite == null || edek == null) {
- throw new RetryStartFileException();
- }
- // Path is within an EZ and we have provided encryption parameters.
- // Make sure that the generated EDEK matches the settings of the EZ.
- final String ezKeyName = zone.getKeyName();
- if (!ezKeyName.equals(edek.getEncryptionKeyName())) {
- throw new RetryStartFileException();
- }
- feInfo = new FileEncryptionInfo(suite, version,
- edek.getEncryptedKeyVersion().getMaterial(),
- edek.getEncryptedKeyIv(),
- ezKeyName, edek.getEncryptionKeyVersionName());
- }
-
- try {
- BlocksMapUpdateInfo toRemoveBlocks = null;
- if (myFile == null) {
- if (!create) {
- throw new FileNotFoundException("Can't overwrite non-existent " +
- src + " for client " + clientMachine);
- }
- } else {
- if (overwrite) {
- toRemoveBlocks = new BlocksMapUpdateInfo();
- List<INode> toRemoveINodes = new ChunkedArrayList<>();
- List<Long> toRemoveUCFiles = new ChunkedArrayList<>();
- long ret = FSDirDeleteOp.delete(
- dir, iip, toRemoveBlocks, toRemoveINodes,
- toRemoveUCFiles, now());
- if (ret >= 0) {
- iip = INodesInPath.replace(iip, iip.length() - 1, null);
- FSDirDeleteOp.incrDeletedFileCount(ret);
- removeLeasesAndINodes(toRemoveUCFiles, toRemoveINodes, true);
- }
- } else {
- // If lease soft limit time is expired, recover the lease
- recoverLeaseInternal(RecoverLeaseOp.CREATE_FILE,
- iip, src, holder, clientMachine, false);
- throw new FileAlreadyExistsException(src + " for client " +
- clientMachine + " already exists");
- }
- }
-
- checkFsObjectLimit();
- INodeFile newNode = null;
-
- // Always do an implicit mkdirs for parent directory tree.
- Map.Entry<INodesInPath, String> parent = FSDirMkdirOp
- .createAncestorDirectories(dir, iip, permissions);
- if (parent != null) {
- iip = dir.addFile(parent.getKey(), parent.getValue(), permissions,
- replication, blockSize, holder, clientMachine);
- newNode = iip != null ? iip.getLastINode().asFile() : null;
- }
-
- if (newNode == null) {
- throw new IOException("Unable to add " + src + " to namespace");
- }
- leaseManager.addLease(newNode.getFileUnderConstructionFeature()
- .getClientName(), newNode.getId());
-
- // Set encryption attributes if necessary
- if (feInfo != null) {
- dir.setFileEncryptionInfo(src, feInfo);
- newNode = dir.getInode(newNode.getId()).asFile();
- }
-
- setNewINodeStoragePolicy(newNode, iip, isLazyPersist);
-
- // record file record in log, record new generation stamp
- getEditLog().logOpenFile(src, newNode, overwrite, logRetryEntry);
- if (NameNode.stateChangeLog.isDebugEnabled()) {
- NameNode.stateChangeLog.debug("DIR* NameSystem.startFile: added " +
- src + " inode " + newNode.getId() + " " + holder);
- }
- return toRemoveBlocks;
- } catch (IOException ie) {
- NameNode.stateChangeLog.warn("DIR* NameSystem.startFile: " + src + " " +
- ie.getMessage());
- throw ie;
- }
- }
-
- private void setNewINodeStoragePolicy(INodeFile inode,
- INodesInPath iip,
- boolean isLazyPersist)
- throws IOException {
-
- if (isLazyPersist) {
- BlockStoragePolicy lpPolicy =
- blockManager.getStoragePolicy("LAZY_PERSIST");
-
- // Set LAZY_PERSIST storage policy if the flag was passed to
- // CreateFile.
- if (lpPolicy == null) {
- throw new HadoopIllegalArgumentException(
- "The LAZY_PERSIST storage policy has been disabled " +
- "by the administrator.");
- }
- inode.setStoragePolicyID(lpPolicy.getId(),
- iip.getLatestSnapshotId());
- } else {
- BlockStoragePolicy effectivePolicy =
- blockManager.getStoragePolicy(inode.getStoragePolicyID());
-
- if (effectivePolicy != null &&
- effectivePolicy.isCopyOnCreateFile()) {
- // Copy effective policy from ancestor directory to current file.
- inode.setStoragePolicyID(effectivePolicy.getId(),
- iip.getLatestSnapshotId());
- }
- }
- }
-
- /**
* Append to an existing file for append.
* <p>
*
@@ -2871,7 +2671,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return false;
}
- private enum RecoverLeaseOp {
+ enum RecoverLeaseOp {
CREATE_FILE,
APPEND_FILE,
TRUNCATE_FILE,
[48/50] [abbrv] hadoop git commit: HDFS-4383. Document the lease
limits. Contributed by Arshad Mohammad.
Posted by zj...@apache.org.
HDFS-4383. Document the lease limits. Contributed by Arshad Mohammad.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e6cc8c4d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e6cc8c4d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e6cc8c4d
Branch: refs/heads/YARN-2928
Commit: e6cc8c4d1285029d52d7234f06cf3b13fa83abb7
Parents: af9c559
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu May 21 17:30:43 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 12:42:58 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../hdfs/server/common/HdfsServerConstants.java | 23 +++++++++++++++++---
2 files changed, 22 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6cc8c4d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 5bcaddd..77d7369 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -575,6 +575,8 @@ Release 2.8.0 - UNRELEASED
HDFS-4185. Add a metric for number of active leases (Rakesh R via raviprak)
+ HDFS-4383. Document the lease limits. (Arshad Mohammad via aajisaka)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6cc8c4d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
index c664b01..26a7ab3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
@@ -42,10 +42,27 @@ import org.apache.hadoop.util.StringUtils;
@InterfaceAudience.Private
public interface HdfsServerConstants {
int MIN_BLOCKS_FOR_WRITE = 1;
- //
- // Timeouts, constants
- //
+ /**
+ * For a HDFS client to write to a file, a lease is granted; During the lease
+ * period, no other client can write to the file. The writing client can
+ * periodically renew the lease. When the file is closed, the lease is
+ * revoked. The lease duration is bound by this soft limit and a
+ * {@link HdfsServerConstants#LEASE_HARDLIMIT_PERIOD hard limit}. Until the
+ * soft limit expires, the writer has sole write access to the file. If the
+ * soft limit expires and the client fails to close the file or renew the
+ * lease, another client can preempt the lease.
+ */
long LEASE_SOFTLIMIT_PERIOD = 60 * 1000;
+ /**
+ * For a HDFS client to write to a file, a lease is granted; During the lease
+ * period, no other client can write to the file. The writing client can
+ * periodically renew the lease. When the file is closed, the lease is
+ * revoked. The lease duration is bound by a
+ * {@link HdfsServerConstants#LEASE_SOFTLIMIT_PERIOD soft limit} and this hard
+ * limit. If after the hard limit expires and the client has failed to renew
+ * the lease, HDFS assumes that the client has quit and will automatically
+ * close the file on behalf of the writer, and recover the lease.
+ */
long LEASE_HARDLIMIT_PERIOD = 60 * LEASE_SOFTLIMIT_PERIOD;
long LEASE_RECOVER_PERIOD = 10 * 1000; // in ms
// We need to limit the length and depth of a path in the filesystem.
[15/50] [abbrv] hadoop git commit: HDFS-8405. Fix a typo in
NamenodeFsck. Contributed by Takanobu Asanuma
Posted by zj...@apache.org.
HDFS-8405. Fix a typo in NamenodeFsck. Contributed by Takanobu Asanuma
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aacdfcb7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aacdfcb7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aacdfcb7
Branch: refs/heads/YARN-2928
Commit: aacdfcb74969486fb1cfd73fca2a7314a13a9ac1
Parents: 6a12018
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Tue May 19 02:57:54 2015 +0800
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:47:03 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../hadoop/hdfs/server/namenode/FsckServlet.java | 2 +-
.../hadoop/hdfs/server/namenode/NamenodeFsck.java | 18 +++++++-----------
.../hadoop/hdfs/server/namenode/TestFsck.java | 8 ++++----
4 files changed, 14 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aacdfcb7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4270a9c..7fd3495 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -868,6 +868,8 @@ Release 2.7.1 - UNRELEASED
HDFS-6300. Prevent multiple balancers from running simultaneously
(Rakesh R via vinayakumarb)
+ HDFS-8405. Fix a typo in NamenodeFsck. (Takanobu Asanuma via szetszwo)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aacdfcb7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java
index 6fb3d21..5fae9cd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FsckServlet.java
@@ -66,7 +66,7 @@ public class FsckServlet extends DfsServlet {
namesystem.getNumberOfDatanodes(DatanodeReportType.LIVE);
new NamenodeFsck(conf, nn,
bm.getDatanodeManager().getNetworkTopology(), pmap, out,
- totalDatanodes, bm.minReplication, remoteAddress).fsck();
+ totalDatanodes, remoteAddress).fsck();
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aacdfcb7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 61f8fdb..44dba28 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -121,7 +121,6 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
private final NameNode namenode;
private final NetworkTopology networktopology;
private final int totalDatanodes;
- private final short minReplication;
private final InetAddress remoteAddress;
private String lostFound = null;
@@ -181,19 +180,17 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
* @param pmap key=value[] map passed to the http servlet as url parameters
* @param out output stream to write the fsck output
* @param totalDatanodes number of live datanodes
- * @param minReplication minimum replication
* @param remoteAddress source address of the fsck request
*/
NamenodeFsck(Configuration conf, NameNode namenode,
NetworkTopology networktopology,
Map<String,String[]> pmap, PrintWriter out,
- int totalDatanodes, short minReplication, InetAddress remoteAddress) {
+ int totalDatanodes, InetAddress remoteAddress) {
this.conf = conf;
this.namenode = namenode;
this.networktopology = networktopology;
this.out = out;
this.totalDatanodes = totalDatanodes;
- this.minReplication = minReplication;
this.remoteAddress = remoteAddress;
this.bpPolicy = BlockPlacementPolicy.getInstance(conf, null,
networktopology,
@@ -308,7 +305,6 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
final long startTime = Time.monotonicNow();
try {
if(blockIds != null) {
-
String[] blocks = blockIds.split(" ");
StringBuilder sb = new StringBuilder();
sb.append("FSCK started by " +
@@ -561,7 +557,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
res.numExpectedReplicas += targetFileReplication;
// count under min repl'd blocks
- if(totalReplicasPerBlock < minReplication){
+ if(totalReplicasPerBlock < res.minReplication){
res.numUnderMinReplicatedBlocks++;
}
@@ -582,7 +578,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
}
// count minimally replicated blocks
- if (totalReplicasPerBlock >= minReplication)
+ if (totalReplicasPerBlock >= res.minReplication)
res.numMinReplicatedBlocks++;
// count missing replicas / under replicated blocks
@@ -601,7 +597,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
decommissioningReplicas + " decommissioning replica(s).");
}
- // count mis replicated blocks block
+ // count mis replicated blocks
BlockPlacementStatus blockPlacementStatus = bpPolicy
.verifyBlockPlacement(path, lBlk, targetFileReplication);
if (!blockPlacementStatus.isPlacementPolicySatisfied()) {
@@ -724,8 +720,8 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
private void countStorageTypeSummary(HdfsFileStatus file, LocatedBlock lBlk) {
StorageType[] storageTypes = lBlk.getStorageTypes();
storageTypeSummary.add(Arrays.copyOf(storageTypes, storageTypes.length),
- namenode.getNamesystem().getBlockManager()
- .getStoragePolicy(file.getStoragePolicy()));
+ namenode.getNamesystem().getBlockManager()
+ .getStoragePolicy(file.getStoragePolicy()));
}
private void deleteCorruptedFile(String path) {
@@ -1069,7 +1065,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
((float) (numUnderMinReplicatedBlocks * 100) / (float) totalBlocks))
.append(" %)");
}
- res.append("\n ").append("DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY:\t")
+ res.append("\n ").append(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY + ":\t")
.append(minReplication);
}
if(corruptFiles>0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aacdfcb7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index 1ce09e1..7f31f84 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -785,7 +785,7 @@ public class TestFsck {
System.out.println(outStr);
assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
assertTrue(outStr.contains("UNDER MIN REPL'D BLOCKS:\t1 (100.0 %)"));
- assertTrue(outStr.contains("DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY:\t2"));
+ assertTrue(outStr.contains("dfs.namenode.replication.min:\t2"));
} finally {
if (cluster != null) {cluster.shutdown();}
}
@@ -1052,7 +1052,7 @@ public class TestFsck {
PrintWriter out = new PrintWriter(result, true);
InetAddress remoteAddress = InetAddress.getLocalHost();
NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out,
- NUM_REPLICAS, (short)1, remoteAddress);
+ NUM_REPLICAS, remoteAddress);
// Run the fsck and check the Result
final HdfsFileStatus file =
@@ -1129,7 +1129,7 @@ public class TestFsck {
PrintWriter out = new PrintWriter(result, true);
InetAddress remoteAddress = InetAddress.getLocalHost();
NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out,
- NUM_DN, REPL_FACTOR, remoteAddress);
+ NUM_DN, remoteAddress);
// Run the fsck and check the Result
final HdfsFileStatus file =
@@ -1176,7 +1176,7 @@ public class TestFsck {
when(blockManager.getDatanodeManager()).thenReturn(dnManager);
NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out,
- NUM_REPLICAS, (short)1, remoteAddress);
+ NUM_REPLICAS, remoteAddress);
String pathString = "/tmp/testFile";
[20/50] [abbrv] hadoop git commit: HDFS-6348. SecondaryNameNode not
terminating properly on runtime exceptions (Contributed by Rakesh R)
Posted by zj...@apache.org.
HDFS-6348. SecondaryNameNode not terminating properly on runtime exceptions (Contributed by Rakesh R)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8c1572e1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8c1572e1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8c1572e1
Branch: refs/heads/YARN-2928
Commit: 8c1572e1e59f2e7acd995fd60bcb8f73dbcde953
Parents: f6d1979
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue May 19 12:24:25 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:52:07 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 5 ++-
.../hdfs/server/namenode/SecondaryNameNode.java | 33 ++++++++++----------
.../hdfs/server/namenode/TestStartup.java | 18 ++++++++++-
3 files changed, 37 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c1572e1/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 35c3b5a..e5fcba2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1,4 +1,4 @@
-Hadoop HDFS Change Log
+ Hadoop HDFS Change Log
Trunk (Unreleased)
@@ -788,6 +788,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8403. Eliminate retries in TestFileCreation
#testOverwriteOpenForWrite. (Arpit Agarwal via wheat9)
+ HDFS-6348. SecondaryNameNode not terminating properly on runtime exceptions
+ (Rakesh R via vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c1572e1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
index b499e74..0fa1cd5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
@@ -667,29 +667,28 @@ public class SecondaryNameNode implements Runnable,
opts.usage();
System.exit(0);
}
-
- StringUtils.startupShutdownMessage(SecondaryNameNode.class, argv, LOG);
- Configuration tconf = new HdfsConfiguration();
- SecondaryNameNode secondary = null;
+
try {
+ StringUtils.startupShutdownMessage(SecondaryNameNode.class, argv, LOG);
+ Configuration tconf = new HdfsConfiguration();
+ SecondaryNameNode secondary = null;
secondary = new SecondaryNameNode(tconf, opts);
- } catch (IOException ioe) {
- LOG.fatal("Failed to start secondary namenode", ioe);
- terminate(1);
- }
- if (opts != null && opts.getCommand() != null) {
- int ret = secondary.processStartupCommand(opts);
- terminate(ret);
- }
+ if (opts != null && opts.getCommand() != null) {
+ int ret = secondary.processStartupCommand(opts);
+ terminate(ret);
+ }
- if (secondary != null) {
- secondary.startCheckpointThread();
- secondary.join();
+ if (secondary != null) {
+ secondary.startCheckpointThread();
+ secondary.join();
+ }
+ } catch (Throwable e) {
+ LOG.fatal("Failed to start secondary namenode", e);
+ terminate(1);
}
}
-
-
+
public void startCheckpointThread() {
Preconditions.checkState(checkpointThread == null,
"Should not already have a thread");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c1572e1/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
index 01621ad..4d3cb75 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
@@ -60,6 +60,8 @@ import org.apache.hadoop.hdfs.util.MD5FileUtils;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.PathUtils;
+import org.apache.hadoop.util.ExitUtil.ExitException;
+import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.StringUtils;
import org.apache.log4j.Logger;
import org.junit.After;
@@ -87,6 +89,8 @@ public class TestStartup {
@Before
public void setUp() throws Exception {
+ ExitUtil.disableSystemExit();
+ ExitUtil.resetFirstExitException();
config = new HdfsConfiguration();
hdfsDir = new File(MiniDFSCluster.getBaseDirectory());
@@ -403,7 +407,19 @@ public class TestStartup {
cluster.shutdown();
}
}
-
+
+ @Test(timeout = 30000)
+ public void testSNNStartupWithRuntimeException() throws Exception {
+ String[] argv = new String[] { "-checkpoint" };
+ try {
+ SecondaryNameNode.main(argv);
+ fail("Failed to handle runtime exceptions during SNN startup!");
+ } catch (ExitException ee) {
+ GenericTestUtils.assertExceptionContains("ExitException", ee);
+ assertTrue("Didn't termiated properly ", ExitUtil.terminateCalled());
+ }
+ }
+
@Test
public void testCompression() throws IOException {
LOG.info("Test compressing image.");
[40/50] [abbrv] hadoop git commit: HADOOP-11698. Remove DistCpV1 and
Logalyzer. Contributed by Brahma Reddy Battula.
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/850cc678/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java b/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
deleted file mode 100644
index 20b8ee2..0000000
--- a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
+++ /dev/null
@@ -1,1077 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.tools;
-
-import java.io.ByteArrayOutputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.net.URI;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.StringTokenizer;
-
-import junit.framework.TestCase;
-
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.ContentSummary;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FsShell;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.MiniDFSCluster.Builder;
-import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.log4j.Level;
-
-
-/**
- * A JUnit test for copying files recursively.
- */
-@SuppressWarnings("deprecation")
-public class TestCopyFiles extends TestCase {
- {
- ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.StateChange")
- ).getLogger().setLevel(Level.ERROR);
- ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ERROR);
- ((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.ERROR);
- ((Log4JLogger)DistCpV1.LOG).getLogger().setLevel(Level.ALL);
- }
-
- static final URI LOCAL_FS = URI.create("file:///");
-
- private static final Random RAN = new Random();
- private static final int NFILES = 20;
- private static String TEST_ROOT_DIR =
- new Path(System.getProperty("test.build.data","/tmp"))
- .toString().replace(' ', '+');
-
- /** class MyFile contains enough information to recreate the contents of
- * a single file.
- */
- private static class MyFile {
- private static Random gen = new Random();
- private static final int MAX_LEVELS = 3;
- private static final int MAX_SIZE = 8*1024;
- private static String[] dirNames = {
- "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine"
- };
- private final String name;
- private int size = 0;
- private long seed = 0L;
-
- MyFile() {
- this(gen.nextInt(MAX_LEVELS));
- }
- MyFile(int nLevels) {
- String xname = "";
- if (nLevels != 0) {
- int[] levels = new int[nLevels];
- for (int idx = 0; idx < nLevels; idx++) {
- levels[idx] = gen.nextInt(10);
- }
- StringBuffer sb = new StringBuffer();
- for (int idx = 0; idx < nLevels; idx++) {
- sb.append(dirNames[levels[idx]]);
- sb.append("/");
- }
- xname = sb.toString();
- }
- long fidx = gen.nextLong() & Long.MAX_VALUE;
- name = xname + Long.toString(fidx);
- reset();
- }
- void reset() {
- final int oldsize = size;
- do { size = gen.nextInt(MAX_SIZE); } while (oldsize == size);
- final long oldseed = seed;
- do { seed = gen.nextLong() & Long.MAX_VALUE; } while (oldseed == seed);
- }
- String getName() { return name; }
- int getSize() { return size; }
- long getSeed() { return seed; }
- }
-
- private static MyFile[] createFiles(URI fsname, String topdir)
- throws IOException {
- return createFiles(FileSystem.get(fsname, new Configuration()), topdir);
- }
-
- /** create NFILES with random names and directory hierarchies
- * with random (but reproducible) data in them.
- */
- private static MyFile[] createFiles(FileSystem fs, String topdir)
- throws IOException {
- Path root = new Path(topdir);
- MyFile[] files = new MyFile[NFILES];
- for (int i = 0; i < NFILES; i++) {
- files[i] = createFile(root, fs);
- }
- return files;
- }
-
- static MyFile createFile(Path root, FileSystem fs, int levels)
- throws IOException {
- MyFile f = levels < 0 ? new MyFile() : new MyFile(levels);
- Path p = new Path(root, f.getName());
- FSDataOutputStream out = fs.create(p);
- byte[] toWrite = new byte[f.getSize()];
- new Random(f.getSeed()).nextBytes(toWrite);
- out.write(toWrite);
- out.close();
- FileSystem.LOG.info("created: " + p + ", size=" + f.getSize());
- return f;
- }
-
- static MyFile createFile(Path root, FileSystem fs) throws IOException {
- return createFile(root, fs, -1);
- }
-
- private static boolean checkFiles(FileSystem fs, String topdir, MyFile[] files
- ) throws IOException {
- return checkFiles(fs, topdir, files, false);
- }
-
- private static boolean checkFiles(FileSystem fs, String topdir, MyFile[] files,
- boolean existingOnly) throws IOException {
- Path root = new Path(topdir);
-
- for (int idx = 0; idx < files.length; idx++) {
- Path fPath = new Path(root, files[idx].getName());
- try {
- fs.getFileStatus(fPath);
- FSDataInputStream in = fs.open(fPath);
- byte[] toRead = new byte[files[idx].getSize()];
- byte[] toCompare = new byte[files[idx].getSize()];
- Random rb = new Random(files[idx].getSeed());
- rb.nextBytes(toCompare);
- assertEquals("Cannnot read file.", toRead.length, in.read(toRead));
- in.close();
- for (int i = 0; i < toRead.length; i++) {
- if (toRead[i] != toCompare[i]) {
- return false;
- }
- }
- toRead = null;
- toCompare = null;
- }
- catch(FileNotFoundException fnfe) {
- if (!existingOnly) {
- throw fnfe;
- }
- }
- }
-
- return true;
- }
-
- private static void updateFiles(FileSystem fs, String topdir, MyFile[] files,
- int nupdate) throws IOException {
- assert nupdate <= NFILES;
-
- Path root = new Path(topdir);
-
- for (int idx = 0; idx < nupdate; ++idx) {
- Path fPath = new Path(root, files[idx].getName());
- // overwrite file
- assertTrue(fPath.toString() + " does not exist", fs.exists(fPath));
- FSDataOutputStream out = fs.create(fPath);
- files[idx].reset();
- byte[] toWrite = new byte[files[idx].getSize()];
- Random rb = new Random(files[idx].getSeed());
- rb.nextBytes(toWrite);
- out.write(toWrite);
- out.close();
- }
- }
-
- private static FileStatus[] getFileStatus(FileSystem fs,
- String topdir, MyFile[] files) throws IOException {
- return getFileStatus(fs, topdir, files, false);
- }
- private static FileStatus[] getFileStatus(FileSystem fs,
- String topdir, MyFile[] files, boolean existingOnly) throws IOException {
- Path root = new Path(topdir);
- List<FileStatus> statuses = new ArrayList<FileStatus>();
- for (int idx = 0; idx < NFILES; ++idx) {
- try {
- statuses.add(fs.getFileStatus(new Path(root, files[idx].getName())));
- } catch(FileNotFoundException fnfe) {
- if (!existingOnly) {
- throw fnfe;
- }
- }
- }
- return statuses.toArray(new FileStatus[statuses.size()]);
- }
-
- private static boolean checkUpdate(FileSystem fs, FileStatus[] old,
- String topdir, MyFile[] upd, final int nupdate) throws IOException {
- Path root = new Path(topdir);
-
- // overwrote updated files
- for (int idx = 0; idx < nupdate; ++idx) {
- final FileStatus stat =
- fs.getFileStatus(new Path(root, upd[idx].getName()));
- if (stat.getModificationTime() <= old[idx].getModificationTime()) {
- return false;
- }
- }
- // did not overwrite files not updated
- for (int idx = nupdate; idx < NFILES; ++idx) {
- final FileStatus stat =
- fs.getFileStatus(new Path(root, upd[idx].getName()));
- if (stat.getModificationTime() != old[idx].getModificationTime()) {
- return false;
- }
- }
- return true;
- }
-
- /** delete directory and everything underneath it.*/
- private static void deldir(FileSystem fs, String topdir) throws IOException {
- fs.delete(new Path(topdir), true);
- }
-
- /** copy files from local file system to local file system */
- @SuppressWarnings("deprecation")
- public void testCopyFromLocalToLocal() throws Exception {
- Configuration conf = new Configuration();
- FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
- MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
- ToolRunner.run(new DistCpV1(new Configuration()),
- new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
- "file:///"+TEST_ROOT_DIR+"/destdat"});
- assertTrue("Source and destination directories do not match.",
- checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files));
- deldir(localfs, TEST_ROOT_DIR+"/destdat");
- deldir(localfs, TEST_ROOT_DIR+"/srcdat");
- }
-
- /** copy files from dfs file system to dfs file system */
- @SuppressWarnings("deprecation")
- public void testCopyFromDfsToDfs() throws Exception {
- String namenode = null;
- MiniDFSCluster cluster = null;
- try {
- Configuration conf = new Configuration();
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
- final FileSystem hdfs = cluster.getFileSystem();
- namenode = FileSystem.getDefaultUri(conf).toString();
- if (namenode.startsWith("hdfs://")) {
- MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
- ToolRunner.run(new DistCpV1(conf), new String[] {
- "-log",
- namenode+"/logs",
- namenode+"/srcdat",
- namenode+"/destdat"});
- assertTrue("Source and destination directories do not match.",
- checkFiles(hdfs, "/destdat", files));
- FileSystem fs = FileSystem.get(URI.create(namenode+"/logs"), conf);
- assertTrue("Log directory does not exist.",
- fs.exists(new Path(namenode+"/logs")));
- deldir(hdfs, "/destdat");
- deldir(hdfs, "/srcdat");
- deldir(hdfs, "/logs");
- }
- } finally {
- if (cluster != null) { cluster.shutdown(); }
- }
- }
-
- /** copy empty directory on dfs file system */
- @SuppressWarnings("deprecation")
- public void testEmptyDir() throws Exception {
- String namenode = null;
- MiniDFSCluster cluster = null;
- try {
- Configuration conf = new Configuration();
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
- final FileSystem hdfs = cluster.getFileSystem();
- namenode = FileSystem.getDefaultUri(conf).toString();
- if (namenode.startsWith("hdfs://")) {
-
- FileSystem fs = FileSystem.get(URI.create(namenode), new Configuration());
- fs.mkdirs(new Path("/empty"));
-
- ToolRunner.run(new DistCpV1(conf), new String[] {
- "-log",
- namenode+"/logs",
- namenode+"/empty",
- namenode+"/dest"});
- fs = FileSystem.get(URI.create(namenode+"/destdat"), conf);
- assertTrue("Destination directory does not exist.",
- fs.exists(new Path(namenode+"/dest")));
- deldir(hdfs, "/dest");
- deldir(hdfs, "/empty");
- deldir(hdfs, "/logs");
- }
- } finally {
- if (cluster != null) { cluster.shutdown(); }
- }
- }
-
- /** copy files from local file system to dfs file system */
- @SuppressWarnings("deprecation")
- public void testCopyFromLocalToDfs() throws Exception {
- MiniDFSCluster cluster = null;
- try {
- Configuration conf = new Configuration();
- cluster = new MiniDFSCluster.Builder(conf).build();
- final FileSystem hdfs = cluster.getFileSystem();
- final String namenode = hdfs.getUri().toString();
- if (namenode.startsWith("hdfs://")) {
- MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
- ToolRunner.run(new DistCpV1(conf), new String[] {
- "-log",
- namenode+"/logs",
- "file:///"+TEST_ROOT_DIR+"/srcdat",
- namenode+"/destdat"});
- assertTrue("Source and destination directories do not match.",
- checkFiles(cluster.getFileSystem(), "/destdat", files));
- assertTrue("Log directory does not exist.",
- hdfs.exists(new Path(namenode+"/logs")));
- deldir(hdfs, "/destdat");
- deldir(hdfs, "/logs");
- deldir(FileSystem.get(LOCAL_FS, conf), TEST_ROOT_DIR+"/srcdat");
- }
- } finally {
- if (cluster != null) { cluster.shutdown(); }
- }
- }
-
- /** copy files from dfs file system to local file system */
- @SuppressWarnings("deprecation")
- public void testCopyFromDfsToLocal() throws Exception {
- MiniDFSCluster cluster = null;
- try {
- Configuration conf = new Configuration();
- final FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
- cluster = new MiniDFSCluster.Builder(conf).build();
- final FileSystem hdfs = cluster.getFileSystem();
- final String namenode = FileSystem.getDefaultUri(conf).toString();
- if (namenode.startsWith("hdfs://")) {
- MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
- ToolRunner.run(new DistCpV1(conf), new String[] {
- "-log",
- "/logs",
- namenode+"/srcdat",
- "file:///"+TEST_ROOT_DIR+"/destdat"});
- assertTrue("Source and destination directories do not match.",
- checkFiles(localfs, TEST_ROOT_DIR+"/destdat", files));
- assertTrue("Log directory does not exist.",
- hdfs.exists(new Path("/logs")));
- deldir(localfs, TEST_ROOT_DIR+"/destdat");
- deldir(hdfs, "/logs");
- deldir(hdfs, "/srcdat");
- }
- } finally {
- if (cluster != null) { cluster.shutdown(); }
- }
- }
-
- @SuppressWarnings("deprecation")
- public void testCopyDfsToDfsUpdateOverwrite() throws Exception {
- MiniDFSCluster cluster = null;
- try {
- Configuration conf = new Configuration();
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
- final FileSystem hdfs = cluster.getFileSystem();
- final String namenode = hdfs.getUri().toString();
- if (namenode.startsWith("hdfs://")) {
- MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
- ToolRunner.run(new DistCpV1(conf), new String[] {
- "-p",
- "-log",
- namenode+"/logs",
- namenode+"/srcdat",
- namenode+"/destdat"});
- assertTrue("Source and destination directories do not match.",
- checkFiles(hdfs, "/destdat", files));
- FileSystem fs = FileSystem.get(URI.create(namenode+"/logs"), conf);
- assertTrue("Log directory does not exist.",
- fs.exists(new Path(namenode+"/logs")));
-
- FileStatus[] dchkpoint = getFileStatus(hdfs, "/destdat", files);
- final int nupdate = NFILES>>2;
- updateFiles(cluster.getFileSystem(), "/srcdat", files, nupdate);
- deldir(hdfs, "/logs");
-
- ToolRunner.run(new DistCpV1(conf), new String[] {
- "-prbugp", // no t to avoid preserving mod. times
- "-update",
- "-log",
- namenode+"/logs",
- namenode+"/srcdat",
- namenode+"/destdat"});
- assertTrue("Source and destination directories do not match.",
- checkFiles(hdfs, "/destdat", files));
- assertTrue("Update failed to replicate all changes in src",
- checkUpdate(hdfs, dchkpoint, "/destdat", files, nupdate));
-
- deldir(hdfs, "/logs");
- ToolRunner.run(new DistCpV1(conf), new String[] {
- "-prbugp", // no t to avoid preserving mod. times
- "-overwrite",
- "-log",
- namenode+"/logs",
- namenode+"/srcdat",
- namenode+"/destdat"});
- assertTrue("Source and destination directories do not match.",
- checkFiles(hdfs, "/destdat", files));
- assertTrue("-overwrite didn't.",
- checkUpdate(hdfs, dchkpoint, "/destdat", files, NFILES));
-
- deldir(hdfs, "/destdat");
- deldir(hdfs, "/srcdat");
- deldir(hdfs, "/logs");
- }
- } finally {
- if (cluster != null) { cluster.shutdown(); }
- }
- }
-
- @SuppressWarnings("deprecation")
- public void testCopyDfsToDfsUpdateWithSkipCRC() throws Exception {
- MiniDFSCluster cluster = null;
- try {
- Configuration conf = new Configuration();
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
- final FileSystem hdfs = cluster.getFileSystem();
- final String namenode = hdfs.getUri().toString();
-
- FileSystem fs = FileSystem.get(URI.create(namenode), new Configuration());
- // Create two files of the same name, same length but different
- // contents
- final String testfilename = "test";
- final String srcData = "act act act";
- final String destData = "cat cat cat";
-
- if (namenode.startsWith("hdfs://")) {
- deldir(hdfs,"/logs");
-
- Path srcPath = new Path("/srcdat", testfilename);
- Path destPath = new Path("/destdat", testfilename);
- FSDataOutputStream out = fs.create(srcPath, true);
- out.writeUTF(srcData);
- out.close();
-
- out = fs.create(destPath, true);
- out.writeUTF(destData);
- out.close();
-
- // Run with -skipcrccheck option
- ToolRunner.run(new DistCpV1(conf), new String[] {
- "-p",
- "-update",
- "-skipcrccheck",
- "-log",
- namenode+"/logs",
- namenode+"/srcdat",
- namenode+"/destdat"});
-
- // File should not be overwritten
- FSDataInputStream in = hdfs.open(destPath);
- String s = in.readUTF();
- System.out.println("Dest had: " + s);
- assertTrue("Dest got over written even with skip crc",
- s.equalsIgnoreCase(destData));
- in.close();
-
- deldir(hdfs, "/logs");
-
- // Run without the option
- ToolRunner.run(new DistCpV1(conf), new String[] {
- "-p",
- "-update",
- "-log",
- namenode+"/logs",
- namenode+"/srcdat",
- namenode+"/destdat"});
-
- // File should be overwritten
- in = hdfs.open(destPath);
- s = in.readUTF();
- System.out.println("Dest had: " + s);
-
- assertTrue("Dest did not get overwritten without skip crc",
- s.equalsIgnoreCase(srcData));
- in.close();
-
- deldir(hdfs, "/destdat");
- deldir(hdfs, "/srcdat");
- deldir(hdfs, "/logs");
- }
- } finally {
- if (cluster != null) { cluster.shutdown(); }
- }
- }
-
- @SuppressWarnings("deprecation")
- public void testCopyDuplication() throws Exception {
- final FileSystem localfs = FileSystem.get(LOCAL_FS, new Configuration());
- try {
- MyFile[] files = createFiles(localfs, TEST_ROOT_DIR+"/srcdat");
- ToolRunner.run(new DistCpV1(new Configuration()),
- new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
- "file:///"+TEST_ROOT_DIR+"/src2/srcdat"});
- assertTrue("Source and destination directories do not match.",
- checkFiles(localfs, TEST_ROOT_DIR+"/src2/srcdat", files));
-
- assertEquals(DistCpV1.DuplicationException.ERROR_CODE,
- ToolRunner.run(new DistCpV1(new Configuration()),
- new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
- "file:///"+TEST_ROOT_DIR+"/src2/srcdat",
- "file:///"+TEST_ROOT_DIR+"/destdat",}));
- }
- finally {
- deldir(localfs, TEST_ROOT_DIR+"/destdat");
- deldir(localfs, TEST_ROOT_DIR+"/srcdat");
- deldir(localfs, TEST_ROOT_DIR+"/src2");
- }
- }
-
- @SuppressWarnings("deprecation")
- public void testCopySingleFile() throws Exception {
- FileSystem fs = FileSystem.get(LOCAL_FS, new Configuration());
- Path root = new Path(TEST_ROOT_DIR+"/srcdat");
- try {
- MyFile[] files = {createFile(root, fs)};
- //copy a dir with a single file
- ToolRunner.run(new DistCpV1(new Configuration()),
- new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
- "file:///"+TEST_ROOT_DIR+"/destdat"});
- assertTrue("Source and destination directories do not match.",
- checkFiles(fs, TEST_ROOT_DIR+"/destdat", files));
-
- //copy a single file
- String fname = files[0].getName();
- Path p = new Path(root, fname);
- FileSystem.LOG.info("fname=" + fname + ", exists? " + fs.exists(p));
- ToolRunner.run(new DistCpV1(new Configuration()),
- new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat/"+fname,
- "file:///"+TEST_ROOT_DIR+"/dest2/"+fname});
- assertTrue("Source and destination directories do not match.",
- checkFiles(fs, TEST_ROOT_DIR+"/dest2", files));
-
- // single file update should skip copy if destination has the file already
- String[] args = {"-update", "file:///"+TEST_ROOT_DIR+"/srcdat/"+fname,
- "file:///"+TEST_ROOT_DIR+"/dest2/"+fname};
- Configuration conf = new Configuration();
- JobConf job = new JobConf(conf, DistCpV1.class);
- DistCpV1.Arguments distcpArgs = DistCpV1.Arguments.valueOf(args, conf);
- assertFalse("Single file update failed to skip copying even though the "
- + "file exists at destination.", DistCpV1.setup(conf, job, distcpArgs));
-
- //copy single file to existing dir
- deldir(fs, TEST_ROOT_DIR+"/dest2");
- fs.mkdirs(new Path(TEST_ROOT_DIR+"/dest2"));
- MyFile[] files2 = {createFile(root, fs, 0)};
- String sname = files2[0].getName();
- ToolRunner.run(new DistCpV1(new Configuration()),
- new String[] {"-update",
- "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname,
- "file:///"+TEST_ROOT_DIR+"/dest2/"});
- assertTrue("Source and destination directories do not match.",
- checkFiles(fs, TEST_ROOT_DIR+"/dest2", files2));
- updateFiles(fs, TEST_ROOT_DIR+"/srcdat", files2, 1);
- //copy single file to existing dir w/ dst name conflict
- ToolRunner.run(new DistCpV1(new Configuration()),
- new String[] {"-update",
- "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname,
- "file:///"+TEST_ROOT_DIR+"/dest2/"});
- assertTrue("Source and destination directories do not match.",
- checkFiles(fs, TEST_ROOT_DIR+"/dest2", files2));
- }
- finally {
- deldir(fs, TEST_ROOT_DIR+"/destdat");
- deldir(fs, TEST_ROOT_DIR+"/dest2");
- deldir(fs, TEST_ROOT_DIR+"/srcdat");
- }
- }
-
- /** tests basedir option copying files from dfs file system to dfs file system */
- @SuppressWarnings("deprecation")
- public void testBasedir() throws Exception {
- String namenode = null;
- MiniDFSCluster cluster = null;
- try {
- Configuration conf = new Configuration();
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
- final FileSystem hdfs = cluster.getFileSystem();
- namenode = FileSystem.getDefaultUri(conf).toString();
- if (namenode.startsWith("hdfs://")) {
- MyFile[] files = createFiles(URI.create(namenode), "/basedir/middle/srcdat");
- ToolRunner.run(new DistCpV1(conf), new String[] {
- "-basedir",
- "/basedir",
- namenode+"/basedir/middle/srcdat",
- namenode+"/destdat"});
- assertTrue("Source and destination directories do not match.",
- checkFiles(hdfs, "/destdat/middle/srcdat", files));
- deldir(hdfs, "/destdat");
- deldir(hdfs, "/basedir");
- deldir(hdfs, "/logs");
- }
- } finally {
- if (cluster != null) { cluster.shutdown(); }
- }
- }
-
- @SuppressWarnings("deprecation")
- public void testPreserveOption() throws Exception {
- Configuration conf = new Configuration();
- MiniDFSCluster cluster = null;
- try {
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
- String nnUri = FileSystem.getDefaultUri(conf).toString();
- FileSystem fs = FileSystem.get(URI.create(nnUri), conf);
-
- {//test preserving user
- MyFile[] files = createFiles(URI.create(nnUri), "/srcdat");
- FileStatus[] srcstat = getFileStatus(fs, "/srcdat", files);
- for(int i = 0; i < srcstat.length; i++) {
- fs.setOwner(srcstat[i].getPath(), "u" + i, null);
- }
- ToolRunner.run(new DistCpV1(conf),
- new String[]{"-pu", nnUri+"/srcdat", nnUri+"/destdat"});
- assertTrue("Source and destination directories do not match.",
- checkFiles(fs, "/destdat", files));
-
- FileStatus[] dststat = getFileStatus(fs, "/destdat", files);
- for(int i = 0; i < dststat.length; i++) {
- assertEquals("i=" + i, "u" + i, dststat[i].getOwner());
- }
- deldir(fs, "/destdat");
- deldir(fs, "/srcdat");
- }
-
- {//test preserving group
- MyFile[] files = createFiles(URI.create(nnUri), "/srcdat");
- FileStatus[] srcstat = getFileStatus(fs, "/srcdat", files);
- for(int i = 0; i < srcstat.length; i++) {
- fs.setOwner(srcstat[i].getPath(), null, "g" + i);
- }
- ToolRunner.run(new DistCpV1(conf),
- new String[]{"-pg", nnUri+"/srcdat", nnUri+"/destdat"});
- assertTrue("Source and destination directories do not match.",
- checkFiles(fs, "/destdat", files));
-
- FileStatus[] dststat = getFileStatus(fs, "/destdat", files);
- for(int i = 0; i < dststat.length; i++) {
- assertEquals("i=" + i, "g" + i, dststat[i].getGroup());
- }
- deldir(fs, "/destdat");
- deldir(fs, "/srcdat");
- }
-
- {//test preserving mode
- MyFile[] files = createFiles(URI.create(nnUri), "/srcdat");
- FileStatus[] srcstat = getFileStatus(fs, "/srcdat", files);
- FsPermission[] permissions = new FsPermission[srcstat.length];
- for(int i = 0; i < srcstat.length; i++) {
- permissions[i] = new FsPermission((short)(i & 0666));
- fs.setPermission(srcstat[i].getPath(), permissions[i]);
- }
-
- ToolRunner.run(new DistCpV1(conf),
- new String[]{"-pp", nnUri+"/srcdat", nnUri+"/destdat"});
- assertTrue("Source and destination directories do not match.",
- checkFiles(fs, "/destdat", files));
-
- FileStatus[] dststat = getFileStatus(fs, "/destdat", files);
- for(int i = 0; i < dststat.length; i++) {
- assertEquals("i=" + i, permissions[i], dststat[i].getPermission());
- }
- deldir(fs, "/destdat");
- deldir(fs, "/srcdat");
- }
-
- {//test preserving times
- MyFile[] files = createFiles(URI.create(nnUri), "/srcdat");
- fs.mkdirs(new Path("/srcdat/tmpf1"));
- fs.mkdirs(new Path("/srcdat/tmpf2"));
- FileStatus[] srcstat = getFileStatus(fs, "/srcdat", files);
- FsPermission[] permissions = new FsPermission[srcstat.length];
- for(int i = 0; i < srcstat.length; i++) {
- fs.setTimes(srcstat[i].getPath(), 40, 50);
- }
-
- ToolRunner.run(new DistCpV1(conf),
- new String[]{"-pt", nnUri+"/srcdat", nnUri+"/destdat"});
-
- FileStatus[] dststat = getFileStatus(fs, "/destdat", files);
- for(int i = 0; i < dststat.length; i++) {
- assertEquals("Modif. Time i=" + i, 40, dststat[i].getModificationTime());
- assertEquals("Access Time i=" + i+ srcstat[i].getPath() + "-" + dststat[i].getPath(), 50, dststat[i].getAccessTime());
- }
-
- assertTrue("Source and destination directories do not match.",
- checkFiles(fs, "/destdat", files));
-
- deldir(fs, "/destdat");
- deldir(fs, "/srcdat");
- }
- } finally {
- if (cluster != null) { cluster.shutdown(); }
- }
- }
-
- @SuppressWarnings("deprecation")
- public void testMapCount() throws Exception {
- String namenode = null;
- MiniDFSCluster dfs = null;
- MiniDFSCluster mr = null;
- try {
- Configuration conf = new Configuration();
-
- dfs= new MiniDFSCluster.Builder(conf).numDataNodes(3).format(true).build();
-
- FileSystem fs = dfs.getFileSystem();
- final FsShell shell = new FsShell(conf);
- namenode = fs.getUri().toString();
- MyFile[] files = createFiles(fs.getUri(), "/srcdat");
- long totsize = 0;
- for (MyFile f : files) {
- totsize += f.getSize();
- }
-
- Configuration job = new JobConf(conf);
- job.setLong("distcp.bytes.per.map", totsize / 3);
- ToolRunner.run(new DistCpV1(job),
- new String[] {"-m", "100",
- "-log",
- namenode+"/logs",
- namenode+"/srcdat",
- namenode+"/destdat"});
- assertTrue("Source and destination directories do not match.",
- checkFiles(fs, "/destdat", files));
-
- String logdir = namenode + "/logs";
- System.out.println(execCmd(shell, "-lsr", logdir));
- FileStatus[] logs = fs.listStatus(new Path(logdir));
- // rare case where splits are exact, logs.length can be 4
- assertTrue( logs.length == 2);
-
- deldir(fs, "/destdat");
- deldir(fs, "/logs");
- ToolRunner.run(new DistCpV1(job),
- new String[] {"-m", "1",
- "-log",
- namenode+"/logs",
- namenode+"/srcdat",
- namenode+"/destdat"});
-
- System.out.println(execCmd(shell, "-lsr", logdir));
- logs = fs.globStatus(new Path(namenode+"/logs/part*"));
- assertTrue("Unexpected map count, logs.length=" + logs.length,
- logs.length == 1);
- } finally {
- if (dfs != null) { dfs.shutdown(); }
- if (mr != null) { mr.shutdown(); }
- }
- }
-
- @SuppressWarnings("deprecation")
- public void testLimits() throws Exception {
- Configuration conf = new Configuration();
- MiniDFSCluster cluster = null;
- try {
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
- final String nnUri = FileSystem.getDefaultUri(conf).toString();
- final FileSystem fs = FileSystem.get(URI.create(nnUri), conf);
- final DistCpV1 distcp = new DistCpV1(conf);
- final FsShell shell = new FsShell(conf);
-
- final String srcrootdir = "/src_root";
- final Path srcrootpath = new Path(srcrootdir);
- final String dstrootdir = "/dst_root";
- final Path dstrootpath = new Path(dstrootdir);
-
- {//test -filelimit
- MyFile[] files = createFiles(URI.create(nnUri), srcrootdir);
- int filelimit = files.length / 2;
- System.out.println("filelimit=" + filelimit);
-
- ToolRunner.run(distcp,
- new String[]{"-filelimit", ""+filelimit, nnUri+srcrootdir, nnUri+dstrootdir});
- String results = execCmd(shell, "-lsr", dstrootdir);
- results = removePrefix(results, dstrootdir);
- System.out.println("results=" + results);
-
- FileStatus[] dststat = getFileStatus(fs, dstrootdir, files, true);
- assertEquals(filelimit, dststat.length);
- deldir(fs, dstrootdir);
- deldir(fs, srcrootdir);
- }
-
- {//test -sizelimit
- createFiles(URI.create(nnUri), srcrootdir);
- long sizelimit = fs.getContentSummary(srcrootpath).getLength()/2;
- System.out.println("sizelimit=" + sizelimit);
-
- ToolRunner.run(distcp,
- new String[]{"-sizelimit", ""+sizelimit, nnUri+srcrootdir, nnUri+dstrootdir});
-
- ContentSummary summary = fs.getContentSummary(dstrootpath);
- System.out.println("summary=" + summary);
- assertTrue(summary.getLength() <= sizelimit);
- deldir(fs, dstrootdir);
- deldir(fs, srcrootdir);
- }
-
- {//test update
- final MyFile[] srcs = createFiles(URI.create(nnUri), srcrootdir);
- final long totalsize = fs.getContentSummary(srcrootpath).getLength();
- System.out.println("src.length=" + srcs.length);
- System.out.println("totalsize =" + totalsize);
- fs.mkdirs(dstrootpath);
- final int parts = RAN.nextInt(NFILES/3 - 1) + 2;
- final int filelimit = srcs.length/parts;
- final long sizelimit = totalsize/parts;
- System.out.println("filelimit=" + filelimit);
- System.out.println("sizelimit=" + sizelimit);
- System.out.println("parts =" + parts);
- final String[] args = {"-filelimit", ""+filelimit, "-sizelimit", ""+sizelimit,
- "-update", nnUri+srcrootdir, nnUri+dstrootdir};
-
- int dstfilecount = 0;
- long dstsize = 0;
- for(int i = 0; i <= parts; i++) {
- ToolRunner.run(distcp, args);
-
- FileStatus[] dststat = getFileStatus(fs, dstrootdir, srcs, true);
- System.out.println(i + ") dststat.length=" + dststat.length);
- assertTrue(dststat.length - dstfilecount <= filelimit);
- ContentSummary summary = fs.getContentSummary(dstrootpath);
- System.out.println(i + ") summary.getLength()=" + summary.getLength());
- assertTrue(summary.getLength() - dstsize <= sizelimit);
- assertTrue(checkFiles(fs, dstrootdir, srcs, true));
- dstfilecount = dststat.length;
- dstsize = summary.getLength();
- }
-
- deldir(fs, dstrootdir);
- deldir(fs, srcrootdir);
- }
- } finally {
- if (cluster != null) { cluster.shutdown(); }
- }
- }
-
- static final long now = System.currentTimeMillis();
-
- static UserGroupInformation createUGI(String name, boolean issuper) {
- String username = name + now;
- String group = issuper? "supergroup": username;
- return UserGroupInformation.createUserForTesting(username,
- new String[]{group});
- }
-
- static Path createHomeDirectory(FileSystem fs, UserGroupInformation ugi
- ) throws IOException {
- final Path home = new Path("/user/" + ugi.getUserName());
- fs.mkdirs(home);
- fs.setOwner(home, ugi.getUserName(), ugi.getGroupNames()[0]);
- fs.setPermission(home, new FsPermission((short)0700));
- return home;
- }
-
- /** test -delete */
- @SuppressWarnings("deprecation")
- public void testDelete() throws Exception {
- final Configuration conf = new Configuration();
- conf.setInt("fs.trash.interval", 60);
- MiniDFSCluster cluster = null;
- try {
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
- final URI nnURI = FileSystem.getDefaultUri(conf);
- final String nnUri = nnURI.toString();
- final FileSystem fs = FileSystem.get(URI.create(nnUri), conf);
-
- final DistCpV1 distcp = new DistCpV1(conf);
- final FsShell shell = new FsShell(conf);
-
- final String srcrootdir = "/src_root";
- final String dstrootdir = "/dst_root";
-
- {
- //create source files
- createFiles(nnURI, srcrootdir);
- String srcresults = execCmd(shell, "-lsr", srcrootdir);
- srcresults = removePrefix(srcresults, srcrootdir);
- System.out.println("srcresults=" + srcresults);
-
- //create some files in dst
- createFiles(nnURI, dstrootdir);
- System.out.println("dstrootdir=" + dstrootdir);
- shell.run(new String[]{"-lsr", dstrootdir});
-
- //run distcp
- ToolRunner.run(distcp,
- new String[]{"-delete", "-update", "-log", "/log",
- nnUri+srcrootdir, nnUri+dstrootdir});
-
- //make sure src and dst contains the same files
- String dstresults = execCmd(shell, "-lsr", dstrootdir);
- dstresults = removePrefix(dstresults, dstrootdir);
- System.out.println("first dstresults=" + dstresults);
- assertEquals(srcresults, dstresults);
-
- //create additional file in dst
- create(fs, new Path(dstrootdir, "foo"));
- create(fs, new Path(dstrootdir, "foobar"));
-
- //run distcp again
- ToolRunner.run(distcp,
- new String[]{"-delete", "-update", "-log", "/log2",
- nnUri+srcrootdir, nnUri+dstrootdir});
-
- //make sure src and dst contains the same files
- dstresults = execCmd(shell, "-lsr", dstrootdir);
- dstresults = removePrefix(dstresults, dstrootdir);
- System.out.println("second dstresults=" + dstresults);
- assertEquals(srcresults, dstresults);
- // verify that files removed in -delete were moved to the trash
- // regrettably, this test will break if Trash changes incompatibly
- assertTrue(fs.exists(new Path(fs.getHomeDirectory(),
- ".Trash/Current" + dstrootdir + "/foo")));
- assertTrue(fs.exists(new Path(fs.getHomeDirectory(),
- ".Trash/Current" + dstrootdir + "/foobar")));
-
- //cleanup
- deldir(fs, dstrootdir);
- deldir(fs, srcrootdir);
- }
- } finally {
- if (cluster != null) { cluster.shutdown(); }
- }
- }
-
- /**
- * verify that -delete option works for other {@link FileSystem}
- * implementations. See MAPREDUCE-1285 */
- @SuppressWarnings("deprecation")
- public void testDeleteLocal() throws Exception {
- MiniDFSCluster cluster = null;
- try {
- Configuration conf = new Configuration();
- final FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
- cluster = new MiniDFSCluster.Builder(conf).build();
- final FileSystem hdfs = cluster.getFileSystem();
- final String namenode = FileSystem.getDefaultUri(conf).toString();
- if (namenode.startsWith("hdfs://")) {
- MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
- String destdir = TEST_ROOT_DIR + "/destdat";
- MyFile[] localFiles = createFiles(localfs, destdir);
- ToolRunner.run(new DistCpV1(conf), new String[] {
- "-delete",
- "-update",
- "-log",
- "/logs",
- namenode+"/srcdat",
- "file:///"+TEST_ROOT_DIR+"/destdat"});
- assertTrue("Source and destination directories do not match.",
- checkFiles(localfs, destdir, files));
- assertTrue("Log directory does not exist.",
- hdfs.exists(new Path("/logs")));
- deldir(localfs, destdir);
- deldir(hdfs, "/logs");
- deldir(hdfs, "/srcdat");
- }
- } finally {
- if (cluster != null) { cluster.shutdown(); }
- }
- }
-
- /** test globbing */
- @SuppressWarnings("deprecation")
- public void testGlobbing() throws Exception {
- String namenode = null;
- MiniDFSCluster cluster = null;
- try {
- Configuration conf = new Configuration();
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
- final FileSystem hdfs = cluster.getFileSystem();
- namenode = FileSystem.getDefaultUri(conf).toString();
- if (namenode.startsWith("hdfs://")) {
- MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
- ToolRunner.run(new DistCpV1(conf), new String[] {
- "-log",
- namenode+"/logs",
- namenode+"/srcdat/*",
- namenode+"/destdat"});
- assertTrue("Source and destination directories do not match.",
- checkFiles(hdfs, "/destdat", files));
- FileSystem fs = FileSystem.get(URI.create(namenode+"/logs"), conf);
- assertTrue("Log directory does not exist.",
- fs.exists(new Path(namenode+"/logs")));
- deldir(hdfs, "/destdat");
- deldir(hdfs, "/srcdat");
- deldir(hdfs, "/logs");
- }
- } finally {
- if (cluster != null) { cluster.shutdown(); }
- }
- }
-
- static void create(FileSystem fs, Path f) throws IOException {
- FSDataOutputStream out = fs.create(f);
- try {
- byte[] b = new byte[1024 + RAN.nextInt(1024)];
- RAN.nextBytes(b);
- out.write(b);
- } finally {
- if (out != null) out.close();
- }
- }
-
- static String execCmd(FsShell shell, String... args) throws Exception {
- ByteArrayOutputStream baout = new ByteArrayOutputStream();
- PrintStream out = new PrintStream(baout, true);
- PrintStream old = System.out;
- System.setOut(out);
- shell.run(args);
- out.close();
- System.setOut(old);
- return baout.toString();
- }
-
- private static String removePrefix(String lines, String prefix) {
- final int prefixlen = prefix.length();
- final StringTokenizer t = new StringTokenizer(lines, "\n");
- final StringBuffer results = new StringBuffer();
- for(; t.hasMoreTokens(); ) {
- String s = t.nextToken();
- results.append(s.substring(s.indexOf(prefix) + prefixlen) + "\n");
- }
- return results.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/850cc678/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestLogalyzer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestLogalyzer.java b/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestLogalyzer.java
deleted file mode 100644
index 3787f6c..0000000
--- a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestLogalyzer.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.tools;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.EnumSet;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.Path;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class TestLogalyzer {
- private static String EL = System.getProperty("line.separator");
- private static String TAB = "\t";
- private static final Log LOG = LogFactory.getLog(TestLogalyzer.class);
-
- private static File workSpace = new File("target",
- TestLogalyzer.class.getName() + "-workSpace");
- private static File outdir = new File(workSpace.getAbsoluteFile()
- + File.separator + "out");
-
- @Test
- @SuppressWarnings("deprecation")
- public void testLogalyzer() throws Exception {
- Path f = createLogFile();
-
- String[] args = new String[10];
-
- args[0] = "-archiveDir";
- args[1] = f.toString();
- args[2] = "-grep";
- args[3] = "44";
- args[4] = "-sort";
- args[5] = "0";
- args[6] = "-analysis";
- args[7] = outdir.getAbsolutePath();
- args[8] = "-separator";
- args[9] = " ";
-
- Logalyzer.main(args);
- checkResult();
-
- }
-
- private void checkResult() throws Exception {
- File result = new File(outdir.getAbsolutePath() + File.separator
- + "part-00000");
- File success = new File(outdir.getAbsolutePath() + File.separator
- + "_SUCCESS");
- Assert.assertTrue(success.exists());
-
- FileInputStream fis = new FileInputStream(result);
- BufferedReader br = new BufferedReader(new InputStreamReader(fis, "UTF-8"));
- String line = br.readLine();
- Assert.assertTrue(("1 44" + TAB + "2").equals(line));
- line = br.readLine();
-
- Assert.assertTrue(("3 44" + TAB + "1").equals(line));
- line = br.readLine();
-
- Assert.assertTrue(("4 44" + TAB + "1").equals(line));
-
- br.close();
-
- }
-
- /**
- * Create simple log file
- *
- * @return
- * @throws IOException
- */
-
- private Path createLogFile() throws IOException {
-
- FileContext files = FileContext.getLocalFSFileContext();
-
- Path ws = new Path(workSpace.getAbsoluteFile().getAbsolutePath());
-
- files.delete(ws, true);
- Path workSpacePath = new Path(workSpace.getAbsolutePath(), "log");
- files.mkdir(workSpacePath, null, true);
-
- LOG.info("create logfile.log");
- Path logfile1 = new Path(workSpacePath, "logfile.log");
-
- FSDataOutputStream os = files.create(logfile1,
- EnumSet.of(CreateFlag.CREATE));
- os.writeBytes("4 3" + EL + "1 3" + EL + "4 44" + EL);
- os.writeBytes("2 3" + EL + "1 3" + EL + "0 45" + EL);
- os.writeBytes("4 3" + EL + "1 3" + EL + "1 44" + EL);
-
- os.flush();
- os.close();
- LOG.info("create logfile1.log");
-
- Path logfile2 = new Path(workSpacePath, "logfile1.log");
-
- os = files.create(logfile2, EnumSet.of(CreateFlag.CREATE));
- os.writeBytes("4 3" + EL + "1 3" + EL + "3 44" + EL);
- os.writeBytes("2 3" + EL + "1 3" + EL + "0 45" + EL);
- os.writeBytes("4 3" + EL + "1 3" + EL + "1 44" + EL);
-
- os.flush();
- os.close();
-
- return workSpacePath;
- }
-}
[38/50] [abbrv] hadoop git commit: YARN-3677. Fix findbugs warnings
in yarn-server-resourcemanager. Contributed by Vinod Kumar Vavilapalli.
Posted by zj...@apache.org.
YARN-3677. Fix findbugs warnings in yarn-server-resourcemanager. Contributed by Vinod Kumar Vavilapalli.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/31274c1e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/31274c1e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/31274c1e
Branch: refs/heads/YARN-2928
Commit: 31274c1e081c49b581b01bf64ae7272fa25f98de
Parents: c7ff581
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Wed May 20 08:53:50 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 12:42:56 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../apache/hadoop/yarn/server/resourcemanager/RMAppManager.java | 1 +
.../server/resourcemanager/recovery/FileSystemRMStateStore.java | 2 +-
3 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31274c1e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 46a0e51..fcfa124 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -604,6 +604,9 @@ Release 2.7.1 - UNRELEASED
YARN-3601. Fix UT TestRMFailover.testRMWebAppRedirect. (Weiwei Yang via xgong)
+ YARN-3677. Fix findbugs warnings in yarn-server-resourcemanager.
+ (Vinod Kumar Vavilapalli via ozawa)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31274c1e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 432a4c7..d0c79a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -230,6 +230,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
success = true;
break;
default:
+ break;
}
if (success) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31274c1e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
index 6920bb5..0d97d6f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
@@ -100,7 +100,7 @@ public class FileSystemRMStateStore extends RMStateStore {
private Path dtSequenceNumberPath = null;
private int fsNumRetries;
private long fsRetryInterval;
- private boolean isHDFS;
+ private volatile boolean isHDFS;
@VisibleForTesting
Path fsWorkingPath;
[04/50] [abbrv] hadoop git commit: HDFS-8403. Eliminate retries in
TestFileCreation#testOverwriteOpenForWrite. Contributed by Arpit Agarwal.
Posted by zj...@apache.org.
HDFS-8403. Eliminate retries in TestFileCreation#testOverwriteOpenForWrite. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cf050a3e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cf050a3e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cf050a3e
Branch: refs/heads/YARN-2928
Commit: cf050a3ecc148d67e957a28e6ec2cf4f762b7d8b
Parents: 480b65c
Author: Haohui Mai <wh...@apache.org>
Authored: Fri May 15 19:12:40 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:47:00 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/DFSClient.java | 5 +++-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 9 +++++++
.../org/apache/hadoop/hdfs/NameNodeProxies.java | 28 +++++++++++++++++++-
.../apache/hadoop/hdfs/TestFileCreation.java | 10 +++++--
5 files changed, 51 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf050a3e/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4a33987..0c4d850 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -775,6 +775,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8371. Fix test failure in TestHdfsConfigFields for spanreceiver
properties. (Ray Chiang via aajisaka)
+ HDFS-8403. Eliminate retries in TestFileCreation
+ #testOverwriteOpenForWrite. (Arpit Agarwal via wheat9)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf050a3e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 7908451..f32702e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -341,10 +341,13 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
this.namenode = rpcNamenode;
dtService = null;
} else {
+ boolean noRetries = conf.getBoolean(
+ DFSConfigKeys.DFS_CLIENT_TEST_NO_PROXY_RETRIES,
+ DFSConfigKeys.DFS_CLIENT_TEST_NO_PROXY_RETRIES_DEFAULT);
Preconditions.checkArgument(nameNodeUri != null,
"null URI");
proxyInfo = NameNodeProxies.createProxy(conf, nameNodeUri,
- ClientProtocol.class, nnFallbackToSimpleAuth);
+ ClientProtocol.class, nnFallbackToSimpleAuth, !noRetries);
this.dtService = proxyInfo.getDelegationTokenService();
this.namenode = proxyInfo.getProxy();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf050a3e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 1d0cf4b..c903e76 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs;
import java.util.concurrent.TimeUnit;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
@@ -993,6 +994,14 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
// For testing NN retry cache, we can set this property with positive value.
public static final String DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY = "dfs.client.test.drop.namenode.response.number";
public static final int DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT = 0;
+
+ // Create a NN proxy without retries for testing.
+ @VisibleForTesting
+ public static final String DFS_CLIENT_TEST_NO_PROXY_RETRIES =
+ "dfs.client.test.no.proxy.retries";
+ @VisibleForTesting
+ public static final boolean DFS_CLIENT_TEST_NO_PROXY_RETRIES_DEFAULT = false;
+
public static final String DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY =
"dfs.client.slow.io.warning.threshold.ms";
public static final long DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 30000;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf050a3e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
index 77262bc..bafc76f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java
@@ -161,6 +161,31 @@ public class NameNodeProxies {
public static <T> ProxyAndInfo<T> createProxy(Configuration conf,
URI nameNodeUri, Class<T> xface, AtomicBoolean fallbackToSimpleAuth)
throws IOException {
+ return createProxy(conf, nameNodeUri, xface, fallbackToSimpleAuth, true);
+ }
+
+ /**
+ * Creates the namenode proxy with the passed protocol. This will handle
+ * creation of either HA- or non-HA-enabled proxy objects, depending upon
+ * if the provided URI is a configured logical URI.
+ *
+ * @param conf the configuration containing the required IPC
+ * properties, client failover configurations, etc.
+ * @param nameNodeUri the URI pointing either to a specific NameNode
+ * or to a logical nameservice.
+ * @param xface the IPC interface which should be created
+ * @param fallbackToSimpleAuth set to true or false during calls to
+ * indicate if a secure client falls back to simple auth
+ * @param withRetries certain interfaces have a non-standard retry policy
+ * @return an object containing both the proxy and the associated
+ * delegation token service it corresponds to
+ * @throws IOException if there is an error creating the proxy
+ **/
+ @SuppressWarnings("unchecked")
+ public static <T> ProxyAndInfo<T> createProxy(Configuration conf,
+ URI nameNodeUri, Class<T> xface, AtomicBoolean fallbackToSimpleAuth,
+ boolean withRetries)
+ throws IOException {
AbstractNNFailoverProxyProvider<T> failoverProxyProvider =
createFailoverProxyProvider(conf, nameNodeUri, xface, true,
fallbackToSimpleAuth);
@@ -168,7 +193,8 @@ public class NameNodeProxies {
if (failoverProxyProvider == null) {
// Non-HA case
return createNonHAProxy(conf, NameNode.getAddress(nameNodeUri), xface,
- UserGroupInformation.getCurrentUser(), true, fallbackToSimpleAuth);
+ UserGroupInformation.getCurrentUser(), withRetries,
+ fallbackToSimpleAuth);
} else {
// HA case
DfsClientConf config = new DfsClientConf(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf050a3e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
index f732ace..4977015 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -73,6 +74,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
@@ -377,6 +379,11 @@ public class TestFileCreation {
Configuration conf = new HdfsConfiguration();
SimulatedFSDataset.setFactory(conf);
conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, false);
+
+ // Force NameNodeProxies' createNNProxyWithClientProtocol to give
+ // up file creation after one failure.
+ conf.setBoolean(DFSConfigKeys.DFS_CLIENT_TEST_NO_PROXY_RETRIES, true);
+
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
FileSystem fs = cluster.getFileSystem();
@@ -405,9 +412,8 @@ public class TestFileCreation {
} catch (IOException abce) {
GenericTestUtils.assertExceptionContains("Failed to CREATE_FILE", abce);
}
- // NameNodeProxies' createNNProxyWithClientProtocol has 5 retries.
assertCounter("AlreadyBeingCreatedExceptionNumOps",
- 6L, getMetrics(metricsName));
+ 1L, getMetrics(metricsName));
FSDataOutputStream stm2 = fs2.create(p, true);
stm2.write(2);
stm2.close();
[39/50] [abbrv] hadoop git commit: Move YARN-2918 from 2.8.0 to 2.7.1
Posted by zj...@apache.org.
Move YARN-2918 from 2.8.0 to 2.7.1
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f638f3c3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f638f3c3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f638f3c3
Branch: refs/heads/YARN-2928
Commit: f638f3c372711e4b3862593222c609b26a234b3f
Parents: 850cc67
Author: Wangda Tan <wa...@apache.org>
Authored: Wed May 20 13:49:10 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 12:42:56 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f638f3c3/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index fcfa124..93e78af 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -466,9 +466,6 @@ Release 2.8.0 - UNRELEASED
YARN-3584. Fixed attempt diagnostics format shown on the UI. (nijel via
jianhe)
- YARN-2918. RM should not fail on startup if queue's configured labels do
- not exist in cluster-node-labels. (Wangda Tan via jianhe)
-
YARN-1832. Fix wrong MockLocalizerStatus#equals implementation.
(Hong Zhiguo via aajisaka)
@@ -607,6 +604,9 @@ Release 2.7.1 - UNRELEASED
YARN-3677. Fix findbugs warnings in yarn-server-resourcemanager.
(Vinod Kumar Vavilapalli via ozawa)
+ YARN-2918. RM should not fail on startup if queue's configured labels do
+ not exist in cluster-node-labels. (Wangda Tan via jianhe)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
[12/50] [abbrv] hadoop git commit: Updating CHANGES.txt for moving
entry of HDFS-8332 from branch-2 to trunk
Posted by zj...@apache.org.
Updating CHANGES.txt for moving entry of HDFS-8332 from branch-2 to trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e2e9de4c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e2e9de4c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e2e9de4c
Branch: refs/heads/YARN-2928
Commit: e2e9de4c16878c7c92723d0c21daa87ce7e97278
Parents: a43d72f
Author: Uma Maheswara Rao G <um...@apache.org>
Authored: Mon May 18 14:48:49 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:47:02 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2e9de4c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8c823ef..3e0d360 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -330,6 +330,8 @@ Trunk (Unreleased)
HDFS-7673. synthetic load generator docs give incorrect/incomplete commands
(Brahma Reddy Battula via aw)
+ HDFS-8332. DFS client API calls should check filesystem closed (Rakesh R via umamahesh)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -711,8 +713,6 @@ Release 2.8.0 - UNRELEASED
HDFS-6291. FSImage may be left unclosed in BootstrapStandby#doRun()
(Sanghyun Yun via vinayakumarb)
- HDFS-8332. DFS client API calls should check filesystem closed (Rakesh R via umamahesh)
-
HDFS-7998. HDFS Federation : Command mentioned to add a NN to existing
federated cluster is wrong (Ajith S via vinayakumarb)
[19/50] [abbrv] hadoop git commit: HADOOP-1540. Support file
exclusion list in distcp. Contributed by Rich Haase.
Posted by zj...@apache.org.
HADOOP-1540. Support file exclusion list in distcp. Contributed by Rich Haase.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f6d19790
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f6d19790
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f6d19790
Branch: refs/heads/YARN-2928
Commit: f6d19790798a624d11e06261c00d03998cd6c8ee
Parents: 1f5fce6
Author: Jing Zhao <ji...@apache.org>
Authored: Mon May 18 13:24:35 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:52:07 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
.../org/apache/hadoop/tools/CopyFilter.java | 60 +++++
.../apache/hadoop/tools/DistCpConstants.java | 3 +-
.../apache/hadoop/tools/DistCpOptionSwitch.java | 11 +-
.../org/apache/hadoop/tools/DistCpOptions.java | 30 ++-
.../org/apache/hadoop/tools/OptionsParser.java | 267 ++++++++++++-------
.../apache/hadoop/tools/RegexCopyFilter.java | 98 +++++++
.../apache/hadoop/tools/SimpleCopyListing.java | 23 +-
.../org/apache/hadoop/tools/TrueCopyFilter.java | 33 +++
.../org/apache/hadoop/tools/package-info.java | 26 ++
.../apache/hadoop/tools/TestCopyListing.java | 34 ---
.../apache/hadoop/tools/TestIntegration.java | 49 ----
.../apache/hadoop/tools/TestOptionsParser.java | 17 +-
.../hadoop/tools/TestRegexCopyFilter.java | 113 ++++++++
.../apache/hadoop/tools/TestTrueCopyFilter.java | 36 +++
15 files changed, 613 insertions(+), 189 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d19790/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 324434b..cf09c5f 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -583,6 +583,8 @@ Release 2.8.0 - UNRELEASED
HADOOP-11944. add option to test-patch to avoid relocating patch process
directory (Sean Busbey via aw)
+ HADOOP-1540. Support file exclusion list in distcp. (Rich Haase via jing9)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d19790/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyFilter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyFilter.java
new file mode 100644
index 0000000..3da364c
--- /dev/null
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyFilter.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tools;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Interface for excluding files from DistCp.
+ *
+ */
+public abstract class CopyFilter {
+
+ /**
+ * Default initialize method does nothing.
+ */
+ public void initialize() {}
+
+ /**
+ * Predicate to determine if a file can be excluded from copy.
+ *
+ * @param path a Path to be considered for copying
+ * @return boolean, true to copy, false to exclude
+ */
+ public abstract boolean shouldCopy(Path path);
+
+ /**
+ * Public factory method which returns the appropriate implementation of
+ * CopyFilter.
+ *
+ * @param conf DistCp configuratoin
+ * @return An instance of the appropriate CopyFilter
+ */
+ public static CopyFilter getCopyFilter(Configuration conf) {
+ String filtersFilename = conf.get(DistCpConstants.CONF_LABEL_FILTERS_FILE);
+
+ if (filtersFilename == null) {
+ return new TrueCopyFilter();
+ } else {
+ String filterFilename = conf.get(
+ DistCpConstants.CONF_LABEL_FILTERS_FILE);
+ return new RegexCopyFilter(filterFilename);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d19790/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
index 7ecb6ce..21dca62 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
@@ -59,7 +59,8 @@ public class DistCpConstants {
public static final String CONF_LABEL_APPEND = "distcp.copy.append";
public static final String CONF_LABEL_DIFF = "distcp.copy.diff";
public static final String CONF_LABEL_BANDWIDTH_MB = "distcp.map.bandwidth.mb";
-
+ public static final String CONF_LABEL_FILTERS_FILE =
+ "distcp.filters.file";
public static final String CONF_LABEL_MAX_CHUNKS_TOLERABLE =
"distcp.dynamic.max.chunks.tolerable";
public static final String CONF_LABEL_MAX_CHUNKS_IDEAL =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d19790/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
index f90319d..ed4a0b2 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
@@ -177,7 +177,16 @@ public enum DistCpOptionSwitch {
* Specify bandwidth per map in MB
*/
BANDWIDTH(DistCpConstants.CONF_LABEL_BANDWIDTH_MB,
- new Option("bandwidth", true, "Specify bandwidth per map in MB"));
+ new Option("bandwidth", true, "Specify bandwidth per map in MB")),
+
+ /**
+ * Path containing a list of strings, which when found in the path of
+ * a file to be copied excludes that file from the copy job.
+ */
+ FILTERS(DistCpConstants.CONF_LABEL_FILTERS_FILE,
+ new Option("filters", true, "The path to a file containing a list of"
+ + " strings for paths to be excluded from the copy."));
+
public static final String PRESERVE_STATUS_DEFAULT = "-prbugpct";
private final String confLabel;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d19790/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
index d8f3ff7..302b626 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
@@ -69,7 +69,12 @@ public class DistCpOptions {
private Path targetPath;
- // targetPathExist is a derived field, it's initialized in the
+ /**
+ * The path to a file containing a list of paths to filter out of the copy.
+ */
+ private String filtersFile;
+
+ // targetPathExist is a derived field, it's initialized in the
// beginning of distcp.
private boolean targetPathExists = true;
@@ -139,6 +144,7 @@ public class DistCpOptions {
this.sourcePaths = that.getSourcePaths();
this.targetPath = that.getTargetPath();
this.targetPathExists = that.getTargetPathExists();
+ this.filtersFile = that.getFiltersFile();
}
}
@@ -549,6 +555,23 @@ public class DistCpOptions {
return this.targetPathExists = targetPathExists;
}
+ /**
+ * File path that contains the list of patterns
+ * for paths to be filtered from the file copy.
+ * @return - Filter file path.
+ */
+ public final String getFiltersFile() {
+ return filtersFile;
+ }
+
+ /**
+ * Set filtersFile.
+ * @param filtersFilename The path to a list of patterns to exclude from copy.
+ */
+ public final void setFiltersFile(String filtersFilename) {
+ this.filtersFile = filtersFilename;
+ }
+
public void validate(DistCpOptionSwitch option, boolean value) {
boolean syncFolder = (option == DistCpOptionSwitch.SYNC_FOLDERS ?
@@ -623,6 +646,10 @@ public class DistCpOptions {
String.valueOf(mapBandwidth));
DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.PRESERVE_STATUS,
DistCpUtils.packAttributes(preserveStatus));
+ if (filtersFile != null) {
+ DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.FILTERS,
+ filtersFile);
+ }
}
/**
@@ -645,6 +672,7 @@ public class DistCpOptions {
", targetPath=" + targetPath +
", targetPathExists=" + targetPathExists +
", preserveRawXattrs=" + preserveRawXattrs +
+ ", filtersFile='" + filtersFile + '\'' +
'}';
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d19790/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
index 1729479..37add1e 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
@@ -86,37 +86,7 @@ public class OptionsParser {
Arrays.toString(args), e);
}
- DistCpOptions option;
- Path targetPath;
- List<Path> sourcePaths = new ArrayList<Path>();
-
- String leftOverArgs[] = command.getArgs();
- if (leftOverArgs == null || leftOverArgs.length < 1) {
- throw new IllegalArgumentException("Target path not specified");
- }
-
- //Last Argument is the target path
- targetPath = new Path(leftOverArgs[leftOverArgs.length -1].trim());
-
- //Copy any source paths in the arguments to the list
- for (int index = 0; index < leftOverArgs.length - 1; index++) {
- sourcePaths.add(new Path(leftOverArgs[index].trim()));
- }
-
- /* If command has source file listing, use it else, fall back on source paths in args
- If both are present, throw exception and bail */
- if (command.hasOption(DistCpOptionSwitch.SOURCE_FILE_LISTING.getSwitch())) {
- if (!sourcePaths.isEmpty()) {
- throw new IllegalArgumentException("Both source file listing and source paths present");
- }
- option = new DistCpOptions(new Path(getVal(command, DistCpOptionSwitch.
- SOURCE_FILE_LISTING.getSwitch())), targetPath);
- } else {
- if (sourcePaths.isEmpty()) {
- throw new IllegalArgumentException("Neither source file listing nor source paths present");
- }
- option = new DistCpOptions(sourcePaths, targetPath);
- }
+ DistCpOptions option = parseSourceAndTargetPaths(command);
//Process all the other option switches and set options appropriately
if (command.hasOption(DistCpOptionSwitch.IGNORE_FAILURES.getSwitch())) {
@@ -165,68 +135,23 @@ public class OptionsParser {
option.setBlocking(false);
}
- if (command.hasOption(DistCpOptionSwitch.BANDWIDTH.getSwitch())) {
- try {
- Integer mapBandwidth = Integer.parseInt(
- getVal(command, DistCpOptionSwitch.BANDWIDTH.getSwitch()).trim());
- if (mapBandwidth.intValue() <= 0) {
- throw new IllegalArgumentException("Bandwidth specified is not positive: " +
- mapBandwidth);
- }
- option.setMapBandwidth(mapBandwidth);
- } catch (NumberFormatException e) {
- throw new IllegalArgumentException("Bandwidth specified is invalid: " +
- getVal(command, DistCpOptionSwitch.BANDWIDTH.getSwitch()), e);
- }
- }
+ parseBandwidth(command, option);
if (command.hasOption(DistCpOptionSwitch.SSL_CONF.getSwitch())) {
option.setSslConfigurationFile(command.
getOptionValue(DistCpOptionSwitch.SSL_CONF.getSwitch()));
}
- if (command.hasOption(DistCpOptionSwitch.NUM_LISTSTATUS_THREADS.getSwitch())) {
- try {
- Integer numThreads = Integer.parseInt(getVal(command,
- DistCpOptionSwitch.NUM_LISTSTATUS_THREADS.getSwitch()).trim());
- option.setNumListstatusThreads(numThreads);
- } catch (NumberFormatException e) {
- throw new IllegalArgumentException(
- "Number of liststatus threads is invalid: " + getVal(command,
- DistCpOptionSwitch.NUM_LISTSTATUS_THREADS.getSwitch()), e);
- }
- }
+ parseNumListStatusThreads(command, option);
- if (command.hasOption(DistCpOptionSwitch.MAX_MAPS.getSwitch())) {
- try {
- Integer maps = Integer.parseInt(
- getVal(command, DistCpOptionSwitch.MAX_MAPS.getSwitch()).trim());
- option.setMaxMaps(maps);
- } catch (NumberFormatException e) {
- throw new IllegalArgumentException("Number of maps is invalid: " +
- getVal(command, DistCpOptionSwitch.MAX_MAPS.getSwitch()), e);
- }
- }
+ parseMaxMaps(command, option);
if (command.hasOption(DistCpOptionSwitch.COPY_STRATEGY.getSwitch())) {
option.setCopyStrategy(
getVal(command, DistCpOptionSwitch.COPY_STRATEGY.getSwitch()));
}
- if (command.hasOption(DistCpOptionSwitch.PRESERVE_STATUS.getSwitch())) {
- String attributes =
- getVal(command, DistCpOptionSwitch.PRESERVE_STATUS.getSwitch());
- if (attributes == null || attributes.isEmpty()) {
- for (FileAttribute attribute : FileAttribute.values()) {
- option.preserve(attribute);
- }
- } else {
- for (int index = 0; index < attributes.length(); index++) {
- option.preserve(FileAttribute.
- getAttribute(attributes.charAt(index)));
- }
- }
- }
+ parsePreserveStatus(command, option);
if (command.hasOption(DistCpOptionSwitch.DIFF.getSwitch())) {
String[] snapshots = getVals(command, DistCpOptionSwitch.DIFF.getSwitch());
@@ -235,6 +160,47 @@ public class OptionsParser {
option.setUseDiff(true, snapshots[0], snapshots[1]);
}
+ parseFileLimit(command);
+
+ parseSizeLimit(command);
+
+ if (command.hasOption(DistCpOptionSwitch.FILTERS.getSwitch())) {
+ option.setFiltersFile(getVal(command,
+ DistCpOptionSwitch.FILTERS.getSwitch()));
+ }
+
+ return option;
+ }
+
+ /**
+ * parseSizeLimit is a helper method for parsing the deprecated
+ * argument SIZE_LIMIT.
+ *
+ * @param command command line arguments
+ */
+ private static void parseSizeLimit(CommandLine command) {
+ if (command.hasOption(DistCpOptionSwitch.SIZE_LIMIT.getSwitch())) {
+ String sizeLimitString = getVal(command,
+ DistCpOptionSwitch.SIZE_LIMIT.getSwitch().trim());
+ try {
+ Long.parseLong(sizeLimitString);
+ }
+ catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Size-limit is invalid: "
+ + sizeLimitString, e);
+ }
+ LOG.warn(DistCpOptionSwitch.SIZE_LIMIT.getSwitch() + " is a deprecated" +
+ " option. Ignoring.");
+ }
+ }
+
+ /**
+ * parseFileLimit is a helper method for parsing the deprecated
+ * argument FILE_LIMIT.
+ *
+ * @param command command line arguments
+ */
+ private static void parseFileLimit(CommandLine command) {
if (command.hasOption(DistCpOptionSwitch.FILE_LIMIT.getSwitch())) {
String fileLimitString = getVal(command,
DistCpOptionSwitch.FILE_LIMIT.getSwitch().trim());
@@ -246,23 +212,144 @@ public class OptionsParser {
+ fileLimitString, e);
}
LOG.warn(DistCpOptionSwitch.FILE_LIMIT.getSwitch() + " is a deprecated" +
- " option. Ignoring.");
+ " option. Ignoring.");
}
+ }
- if (command.hasOption(DistCpOptionSwitch.SIZE_LIMIT.getSwitch())) {
- String sizeLimitString = getVal(command,
- DistCpOptionSwitch.SIZE_LIMIT.getSwitch().trim());
+ /**
+ * parsePreserveStatus is a helper method for parsing PRESERVE_STATUS.
+ *
+ * @param command command line arguments
+ * @param option parsed distcp options
+ */
+ private static void parsePreserveStatus(CommandLine command,
+ DistCpOptions option) {
+ if (command.hasOption(DistCpOptionSwitch.PRESERVE_STATUS.getSwitch())) {
+ String attributes =
+ getVal(command, DistCpOptionSwitch.PRESERVE_STATUS.getSwitch());
+ if (attributes == null || attributes.isEmpty()) {
+ for (FileAttribute attribute : FileAttribute.values()) {
+ option.preserve(attribute);
+ }
+ } else {
+ for (int index = 0; index < attributes.length(); index++) {
+ option.preserve(FileAttribute.
+ getAttribute(attributes.charAt(index)));
+ }
+ }
+ }
+ }
+
+ /**
+ * parseMaxMaps is a helper method for parsing MAX_MAPS.
+ *
+ * @param command command line arguments
+ * @param option parsed distcp options
+ */
+ private static void parseMaxMaps(CommandLine command,
+ DistCpOptions option) {
+ if (command.hasOption(DistCpOptionSwitch.MAX_MAPS.getSwitch())) {
try {
- Long.parseLong(sizeLimitString);
+ Integer maps = Integer.parseInt(
+ getVal(command, DistCpOptionSwitch.MAX_MAPS.getSwitch()).trim());
+ option.setMaxMaps(maps);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Number of maps is invalid: " +
+ getVal(command, DistCpOptionSwitch.MAX_MAPS.getSwitch()), e);
}
- catch (NumberFormatException e) {
- throw new IllegalArgumentException("Size-limit is invalid: "
- + sizeLimitString, e);
+ }
+ }
+
+ /**
+ * parseNumListStatusThreads is a helper method for parsing
+ * NUM_LISTSTATUS_THREADS.
+ *
+ * @param command command line arguments
+ * @param option parsed distcp options
+ */
+ private static void parseNumListStatusThreads(CommandLine command,
+ DistCpOptions option) {
+ if (command.hasOption(
+ DistCpOptionSwitch.NUM_LISTSTATUS_THREADS.getSwitch())) {
+ try {
+ Integer numThreads = Integer.parseInt(getVal(command,
+ DistCpOptionSwitch.NUM_LISTSTATUS_THREADS.getSwitch()).trim());
+ option.setNumListstatusThreads(numThreads);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(
+ "Number of liststatus threads is invalid: " + getVal(command,
+ DistCpOptionSwitch.NUM_LISTSTATUS_THREADS.getSwitch()), e);
+ }
+ }
+ }
+
+ /**
+ * parseBandwidth is a helper method for parsing BANDWIDTH.
+ *
+ * @param command command line arguments
+ * @param option parsed distcp options
+ */
+ private static void parseBandwidth(CommandLine command,
+ DistCpOptions option) {
+ if (command.hasOption(DistCpOptionSwitch.BANDWIDTH.getSwitch())) {
+ try {
+ Integer mapBandwidth = Integer.parseInt(
+ getVal(command, DistCpOptionSwitch.BANDWIDTH.getSwitch()).trim());
+ if (mapBandwidth <= 0) {
+ throw new IllegalArgumentException("Bandwidth specified is not " +
+ "positive: " + mapBandwidth);
+ }
+ option.setMapBandwidth(mapBandwidth);
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Bandwidth specified is invalid: " +
+ getVal(command, DistCpOptionSwitch.BANDWIDTH.getSwitch()), e);
}
- LOG.warn(DistCpOptionSwitch.SIZE_LIMIT.getSwitch() + " is a deprecated" +
- " option. Ignoring.");
}
+ }
+ /**
+ * parseSourceAndTargetPaths is a helper method for parsing the source
+ * and target paths.
+ *
+ * @param command command line arguments
+ * @return DistCpOptions
+ */
+ private static DistCpOptions parseSourceAndTargetPaths(
+ CommandLine command) {
+ DistCpOptions option;
+ Path targetPath;
+ List<Path> sourcePaths = new ArrayList<Path>();
+
+ String[] leftOverArgs = command.getArgs();
+ if (leftOverArgs == null || leftOverArgs.length < 1) {
+ throw new IllegalArgumentException("Target path not specified");
+ }
+
+ //Last Argument is the target path
+ targetPath = new Path(leftOverArgs[leftOverArgs.length - 1].trim());
+
+ //Copy any source paths in the arguments to the list
+ for (int index = 0; index < leftOverArgs.length - 1; index++) {
+ sourcePaths.add(new Path(leftOverArgs[index].trim()));
+ }
+
+ /* If command has source file listing, use it else, fall back on source
+ paths in args. If both are present, throw exception and bail */
+ if (command.hasOption(
+ DistCpOptionSwitch.SOURCE_FILE_LISTING.getSwitch())) {
+ if (!sourcePaths.isEmpty()) {
+ throw new IllegalArgumentException("Both source file listing and " +
+ "source paths present");
+ }
+ option = new DistCpOptions(new Path(getVal(command, DistCpOptionSwitch.
+ SOURCE_FILE_LISTING.getSwitch())), targetPath);
+ } else {
+ if (sourcePaths.isEmpty()) {
+ throw new IllegalArgumentException("Neither source file listing nor " +
+ "source paths present");
+ }
+ option = new DistCpOptions(sourcePaths, targetPath);
+ }
return option;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d19790/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/RegexCopyFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/RegexCopyFilter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/RegexCopyFilter.java
new file mode 100644
index 0000000..1c2b324
--- /dev/null
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/RegexCopyFilter.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.tools;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.io.IOUtils;
+
+import java.io.*;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * A CopyFilter which compares Java Regex Patterns to each Path to determine
+ * whether a file should be copied.
+ */
+public class RegexCopyFilter extends CopyFilter {
+
+ private static final Log LOG = LogFactory.getLog(RegexCopyFilter.class);
+ private File filtersFile;
+ private List<Pattern> filters;
+
+ /**
+ * Constructor, sets up a File object to read filter patterns from and
+ * the List to store the patterns.
+ */
+ protected RegexCopyFilter(String filtersFilename) {
+ filtersFile = new File(filtersFilename);
+ filters = new ArrayList<>();
+ }
+
+ /**
+ * Loads a list of filter patterns for use in shouldCopy.
+ */
+ @Override
+ public void initialize() {
+ BufferedReader reader = null;
+ try {
+ InputStream is = new FileInputStream(filtersFile);
+ reader = new BufferedReader(new InputStreamReader(is,
+ Charset.forName("UTF-8")));
+ String line;
+ while ((line = reader.readLine()) != null) {
+ Pattern pattern = Pattern.compile(line);
+ filters.add(pattern);
+ }
+ } catch (FileNotFoundException notFound) {
+ LOG.error("Can't find filters file " + filtersFile);
+ } catch (IOException cantRead) {
+ LOG.error("An error occurred while attempting to read from " +
+ filtersFile);
+ } finally {
+ IOUtils.cleanup(LOG, reader);
+ }
+ }
+
+ /**
+ * Sets the list of filters to exclude files from copy.
+ * Simplifies testing of the filters feature.
+ *
+ * @param filtersList a list of Patterns to be excluded
+ */
+ @VisibleForTesting
+ protected final void setFilters(List<Pattern> filtersList) {
+ this.filters = filtersList;
+ }
+
+ @Override
+ public boolean shouldCopy(Path path) {
+ for (Pattern filter : filters) {
+ if (filter.matcher(path.toString()).matches()) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d19790/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
index 4ea1dc9..8f50913 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
@@ -58,6 +58,7 @@ public class SimpleCopyListing extends CopyListing {
private long totalBytesToCopy = 0;
private int numListstatusThreads = 1;
private final int maxRetries = 3;
+ private CopyFilter copyFilter;
/**
* Protected constructor, to initialize configuration.
@@ -71,6 +72,8 @@ public class SimpleCopyListing extends CopyListing {
numListstatusThreads = getConf().getInt(
DistCpConstants.CONF_LABEL_LISTSTATUS_THREADS,
DistCpConstants.DEFAULT_LISTSTATUS_THREADS);
+ copyFilter = CopyFilter.getCopyFilter(getConf());
+ copyFilter.initialize();
}
@VisibleForTesting
@@ -213,7 +216,7 @@ public class SimpleCopyListing extends CopyListing {
preserveXAttrs && sourceStatus.isDirectory(),
preserveRawXAttrs && sourceStatus.isDirectory());
writeToFileListing(fileListWriter, sourceCopyListingStatus,
- sourcePathRoot, options);
+ sourcePathRoot);
if (sourceStatus.isDirectory()) {
if (LOG.isDebugEnabled()) {
@@ -264,11 +267,10 @@ public class SimpleCopyListing extends CopyListing {
* Provide an option to skip copy of a path, Allows for exclusion
* of files such as {@link org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter#SUCCEEDED_FILE_NAME}
* @param path - Path being considered for copy while building the file listing
- * @param options - Input options passed during DistCp invocation
* @return - True if the path should be considered for copy, false otherwise
*/
- protected boolean shouldCopy(Path path, DistCpOptions options) {
- return true;
+ protected boolean shouldCopy(Path path) {
+ return copyFilter.shouldCopy(path);
}
/** {@inheritDoc} */
@@ -409,7 +411,7 @@ public class SimpleCopyListing extends CopyListing {
preserveXAttrs && child.isDirectory(),
preserveRawXattrs && child.isDirectory());
writeToFileListing(fileListWriter, childCopyListingStatus,
- sourcePathRoot, options);
+ sourcePathRoot);
}
if (retry < maxRetries) {
if (child.isDirectory()) {
@@ -443,26 +445,23 @@ public class SimpleCopyListing extends CopyListing {
}
return;
}
- writeToFileListing(fileListWriter, fileStatus, sourcePathRoot, options);
+ writeToFileListing(fileListWriter, fileStatus, sourcePathRoot);
}
private void writeToFileListing(SequenceFile.Writer fileListWriter,
CopyListingFileStatus fileStatus,
- Path sourcePathRoot,
- DistCpOptions options) throws IOException {
+ Path sourcePathRoot) throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("REL PATH: " + DistCpUtils.getRelativePath(sourcePathRoot,
fileStatus.getPath()) + ", FULL PATH: " + fileStatus.getPath());
}
- FileStatus status = fileStatus;
-
- if (!shouldCopy(fileStatus.getPath(), options)) {
+ if (!shouldCopy(fileStatus.getPath())) {
return;
}
fileListWriter.append(new Text(DistCpUtils.getRelativePath(sourcePathRoot,
- fileStatus.getPath())), status);
+ fileStatus.getPath())), fileStatus);
fileListWriter.sync();
if (!fileStatus.isDirectory()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d19790/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/TrueCopyFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/TrueCopyFilter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/TrueCopyFilter.java
new file mode 100644
index 0000000..b58dd9c
--- /dev/null
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/TrueCopyFilter.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.tools;
+
+import org.apache.hadoop.fs.Path;
+
+/**
+ * A CopyFilter which always returns true.
+ *
+ */
+public class TrueCopyFilter extends CopyFilter {
+
+ @Override
+ public boolean shouldCopy(Path path) {
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d19790/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/package-info.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/package-info.java
new file mode 100644
index 0000000..92278ed
--- /dev/null
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/package-info.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ * DistCp is a tool for replicating data using MapReduce jobs for concurrent
+ * copy operations.
+ *
+ * @version 2
+ */
+package org.apache.hadoop.tools;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d19790/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
index 8381c1b..896763d 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
@@ -95,40 +95,6 @@ public class TestCopyListing extends SimpleCopyListing {
}
@Test(timeout=10000)
- public void testSkipCopy() throws Exception {
- SimpleCopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS) {
- @Override
- protected boolean shouldCopy(Path path, DistCpOptions options) {
- return !path.getName().equals(FileOutputCommitter.SUCCEEDED_FILE_NAME);
- }
- };
- FileSystem fs = FileSystem.get(getConf());
- List<Path> srcPaths = new ArrayList<Path>();
- srcPaths.add(new Path("/tmp/in4/1"));
- srcPaths.add(new Path("/tmp/in4/2"));
- Path target = new Path("/tmp/out4/1");
- TestDistCpUtils.createFile(fs, "/tmp/in4/1/_SUCCESS");
- TestDistCpUtils.createFile(fs, "/tmp/in4/1/file");
- TestDistCpUtils.createFile(fs, "/tmp/in4/2");
- fs.mkdirs(target);
- DistCpOptions options = new DistCpOptions(srcPaths, target);
- Path listingFile = new Path("/tmp/list4");
- listing.buildListing(listingFile, options);
- Assert.assertEquals(listing.getNumberOfPaths(), 3);
- SequenceFile.Reader reader = new SequenceFile.Reader(getConf(),
- SequenceFile.Reader.file(listingFile));
- CopyListingFileStatus fileStatus = new CopyListingFileStatus();
- Text relativePath = new Text();
- Assert.assertTrue(reader.next(relativePath, fileStatus));
- Assert.assertEquals(relativePath.toString(), "/1");
- Assert.assertTrue(reader.next(relativePath, fileStatus));
- Assert.assertEquals(relativePath.toString(), "/1/file");
- Assert.assertTrue(reader.next(relativePath, fileStatus));
- Assert.assertEquals(relativePath.toString(), "/2");
- Assert.assertFalse(reader.next(relativePath, fileStatus));
- }
-
- @Test(timeout=10000)
public void testMultipleSrcToFile() {
FileSystem fs = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d19790/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java
index 5726342..ee8e7cc 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java
@@ -242,55 +242,6 @@ public class TestIntegration {
}
@Test(timeout=100000)
- public void testCustomCopyListing() {
-
- try {
- addEntries(listFile, "multifile1/file3", "multifile1/file4", "multifile1/file5");
- createFiles("multifile1/file3", "multifile1/file4", "multifile1/file5");
- mkdirs(target.toString());
-
- Configuration conf = getConf();
- try {
- conf.setClass(DistCpConstants.CONF_LABEL_COPY_LISTING_CLASS,
- CustomCopyListing.class, CopyListing.class);
- DistCpOptions options = new DistCpOptions(Arrays.
- asList(new Path(root + "/" + "multifile1")), target);
- options.setSyncFolder(true);
- options.setDeleteMissing(false);
- options.setOverwrite(false);
- try {
- new DistCp(conf, options).execute();
- } catch (Exception e) {
- LOG.error("Exception encountered ", e);
- throw new IOException(e);
- }
- } finally {
- conf.unset(DistCpConstants.CONF_LABEL_COPY_LISTING_CLASS);
- }
-
- checkResult(target, 2, "file4", "file5");
- } catch (IOException e) {
- LOG.error("Exception encountered while testing distcp", e);
- Assert.fail("distcp failure");
- } finally {
- TestDistCpUtils.delete(fs, root);
- }
- }
-
- private static class CustomCopyListing extends SimpleCopyListing {
-
- public CustomCopyListing(Configuration configuration,
- Credentials credentials) {
- super(configuration, credentials);
- }
-
- @Override
- protected boolean shouldCopy(Path path, DistCpOptions options) {
- return !path.getName().equals("file3");
- }
- }
-
- @Test(timeout=100000)
public void testMultiFileTargetMissing() {
caseMultiFileTargetMissing(false);
caseMultiFileTargetMissing(true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d19790/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
index 6eddfb2..b9d9ada 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
@@ -400,7 +400,7 @@ public class TestOptionsParser {
String val = "DistCpOptions{atomicCommit=false, syncFolder=false, deleteMissing=false, " +
"ignoreFailures=false, maxMaps=20, sslConfigurationFile='null', copyStrategy='uniformsize', " +
"sourceFileListing=abc, sourcePaths=null, targetPath=xyz, targetPathExists=true, " +
- "preserveRawXattrs=false}";
+ "preserveRawXattrs=false, filtersFile='null'}";
Assert.assertEquals(val, option.toString());
Assert.assertNotSame(DistCpOptionSwitch.ATOMIC_COMMIT.toString(),
DistCpOptionSwitch.ATOMIC_COMMIT.name());
@@ -718,4 +718,19 @@ public class TestOptionsParser {
"Diff is valid only with update and delete options", e);
}
}
+
+ @Test
+ public void testExclusionsOption() {
+ DistCpOptions options = OptionsParser.parse(new String[] {
+ "hdfs://localhost:8020/source/first",
+ "hdfs://localhost:8020/target/"});
+ Assert.assertNull(options.getFiltersFile());
+
+ options = OptionsParser.parse(new String[] {
+ "-filters",
+ "/tmp/filters.txt",
+ "hdfs://localhost:8020/source/first",
+ "hdfs://localhost:8020/target/"});
+ Assert.assertEquals(options.getFiltersFile(), "/tmp/filters.txt");
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d19790/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestRegexCopyFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestRegexCopyFilter.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestRegexCopyFilter.java
new file mode 100644
index 0000000..5618a0b
--- /dev/null
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestRegexCopyFilter.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.tools;
+
+import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+
+public class TestRegexCopyFilter {
+
+ @Test
+ public void testShouldCopyTrue() {
+ List<Pattern> filters = new ArrayList<>();
+ filters.add(Pattern.compile("user"));
+
+ RegexCopyFilter regexCopyFilter = new RegexCopyFilter("fakeFile");
+ regexCopyFilter.setFilters(filters);
+
+ Path shouldCopyPath = new Path("/user/bar");
+ Assert.assertTrue(regexCopyFilter.shouldCopy(shouldCopyPath));
+ }
+
+ @Test
+ public void testShouldCopyFalse() {
+ List<Pattern> filters = new ArrayList<>();
+ filters.add(Pattern.compile(".*test.*"));
+
+ RegexCopyFilter regexCopyFilter = new RegexCopyFilter("fakeFile");
+ regexCopyFilter.setFilters(filters);
+
+ Path shouldNotCopyPath = new Path("/user/testing");
+ Assert.assertFalse(regexCopyFilter.shouldCopy(shouldNotCopyPath));
+ }
+
+ @Test
+ public void testShouldCopyWithMultipleFilters() {
+ List<Pattern> filters = new ArrayList<>();
+ filters.add(Pattern.compile(".*test.*"));
+ filters.add(Pattern.compile("/user/b.*"));
+ filters.add(Pattern.compile(".*_SUCCESS"));
+
+ List<Path> toCopy = getTestPaths();
+
+ int shouldCopyCount = 0;
+
+ RegexCopyFilter regexCopyFilter = new RegexCopyFilter("fakeFile");
+ regexCopyFilter.setFilters(filters);
+
+ for (Path path: toCopy) {
+ if (regexCopyFilter.shouldCopy(path)) {
+ shouldCopyCount++;
+ }
+ }
+
+ Assert.assertEquals(2, shouldCopyCount);
+ }
+
+ @Test
+ public void testShouldExcludeAll() {
+ List<Pattern> filters = new ArrayList<>();
+ filters.add(Pattern.compile(".*test.*"));
+ filters.add(Pattern.compile("/user/b.*"));
+ filters.add(Pattern.compile(".*")); // exclude everything
+
+ List<Path> toCopy = getTestPaths();
+
+ int shouldCopyCount = 0;
+
+ RegexCopyFilter regexCopyFilter = new RegexCopyFilter("fakeFile");
+ regexCopyFilter.setFilters(filters);
+
+ for (Path path: toCopy) {
+ if (regexCopyFilter.shouldCopy(path)) {
+ shouldCopyCount++;
+ }
+ }
+
+ Assert.assertEquals(0, shouldCopyCount);
+ }
+
+ private List<Path> getTestPaths() {
+ List<Path> toCopy = new ArrayList<>();
+ toCopy.add(new Path("/user/bar"));
+ toCopy.add(new Path("/user/foo/_SUCCESS"));
+ toCopy.add(new Path("/hive/test_data"));
+ toCopy.add(new Path("test"));
+ toCopy.add(new Path("/user/foo/bar"));
+ toCopy.add(new Path("/mapred/.staging_job"));
+ return toCopy;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6d19790/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestTrueCopyFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestTrueCopyFilter.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestTrueCopyFilter.java
new file mode 100644
index 0000000..2ea60a9
--- /dev/null
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestTrueCopyFilter.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.tools;
+
+import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestTrueCopyFilter {
+
+ @Test
+ public void testShouldCopy() {
+ Assert.assertTrue(new TrueCopyFilter().shouldCopy(new Path("fake")));
+ }
+
+ @Test
+ public void testShouldCopyWithNull() {
+ Assert.assertTrue(new TrueCopyFilter().shouldCopy(new Path("fake")));
+ }
+}
[06/50] [abbrv] hadoop git commit: HADOOP-10582. Fix the test case
for copying to non-existent dir in TestFsShellCopy. Contributed by Kousuke
Saruta.
Posted by zj...@apache.org.
HADOOP-10582. Fix the test case for copying to non-existent dir in TestFsShellCopy. Contributed by Kousuke Saruta.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a43d72fc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a43d72fc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a43d72fc
Branch: refs/heads/YARN-2928
Commit: a43d72fcf034206cf69c0fe680902d0ebe198828
Parents: 24ac9db
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon May 18 16:31:41 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:47:01 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java | 6 +++---
2 files changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a43d72fc/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 7349091..2138334 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -695,6 +695,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11988. Fix typo in the document for hadoop fs -find.
(Kengo Seki via aajisaka)
+ HADOOP-10582. Fix the test case for copying to non-existent dir in
+ TestFsShellCopy. (Kousuke Saruta via aajisaka)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a43d72fc/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java
index bef0c9f..c0a6b20 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java
@@ -177,9 +177,9 @@ public class TestFsShellCopy {
checkPut(0, srcPath, dstPath, useWindowsPath);
}
- // copy to non-existent subdir
- prepPut(childPath, false, false);
- checkPut(1, srcPath, dstPath, useWindowsPath);
+ // copy to non-existent dir
+ prepPut(dstPath, false, false);
+ checkPut(1, srcPath, childPath, useWindowsPath);
// copy into dir, then with another name
prepPut(dstPath, true, true);
[05/50] [abbrv] hadoop git commit: HDFS-8394. Move
getAdditionalBlock() and related functionalities into a separate class.
Contributed by Haohui Mai.
Posted by zj...@apache.org.
HDFS-8394. Move getAdditionalBlock() and related functionalities into a separate class. Contributed by Haohui Mai.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/480b65c7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/480b65c7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/480b65c7
Branch: refs/heads/YARN-2928
Commit: 480b65c71f334a2fa3c317011aea3aad33170014
Parents: c78b397
Author: Haohui Mai <wh...@apache.org>
Authored: Fri May 15 19:09:59 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:47:00 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/namenode/FSDirWriteFileOp.java | 563 +++++++++++++++++++
.../hdfs/server/namenode/FSDirectory.java | 78 +--
.../hdfs/server/namenode/FSEditLogLoader.java | 3 +-
.../hdfs/server/namenode/FSNamesystem.java | 497 ++--------------
.../hdfs/server/namenode/NameNodeRpcServer.java | 30 +-
.../hdfs/server/namenode/TestAddBlockRetry.java | 30 +-
.../TestCommitBlockSynchronization.java | 3 +
8 files changed, 648 insertions(+), 559 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/480b65c7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 35e81f9..4a33987 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -557,6 +557,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8397. Refactor the error handling code in DataStreamer.
(Tsz Wo Nicholas Sze via jing9)
+ HDFS-8394. Move getAdditionalBlock() and related functionalities into a
+ separate class. (wheat9)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/480b65c7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
new file mode 100644
index 0000000..1ff0899
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -0,0 +1,563 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
+import org.apache.hadoop.net.Node;
+import org.apache.hadoop.net.NodeBase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+class FSDirWriteFileOp {
+ private FSDirWriteFileOp() {}
+ static boolean unprotectedRemoveBlock(
+ FSDirectory fsd, String path, INodesInPath iip, INodeFile fileNode,
+ Block block) throws IOException {
+ // modify file-> block and blocksMap
+ // fileNode should be under construction
+ BlockInfoContiguousUnderConstruction uc = fileNode.removeLastBlock(block);
+ if (uc == null) {
+ return false;
+ }
+ fsd.getBlockManager().removeBlockFromMap(block);
+
+ if(NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("DIR* FSDirectory.removeBlock: "
+ +path+" with "+block
+ +" block is removed from the file system");
+ }
+
+ // update space consumed
+ fsd.updateCount(iip, 0, -fileNode.getPreferredBlockSize(),
+ fileNode.getPreferredBlockReplication(), true);
+ return true;
+ }
+
+ /**
+ * Persist the block list for the inode.
+ */
+ static void persistBlocks(
+ FSDirectory fsd, String path, INodeFile file, boolean logRetryCache) {
+ assert fsd.getFSNamesystem().hasWriteLock();
+ Preconditions.checkArgument(file.isUnderConstruction());
+ fsd.getEditLog().logUpdateBlocks(path, file, logRetryCache);
+ if(NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("persistBlocks: " + path
+ + " with " + file.getBlocks().length + " blocks is persisted to" +
+ " the file system");
+ }
+ }
+
+ static void abandonBlock(
+ FSDirectory fsd, FSPermissionChecker pc, ExtendedBlock b, long fileId,
+ String src, String holder) throws IOException {
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ src = fsd.resolvePath(pc, src, pathComponents);
+
+ final INode inode;
+ final INodesInPath iip;
+ if (fileId == HdfsConstants.GRANDFATHER_INODE_ID) {
+ // Older clients may not have given us an inode ID to work with.
+ // In this case, we have to try to resolve the path and hope it
+ // hasn't changed or been deleted since the file was opened for write.
+ iip = fsd.getINodesInPath(src, true);
+ inode = iip.getLastINode();
+ } else {
+ inode = fsd.getInode(fileId);
+ iip = INodesInPath.fromINode(inode);
+ if (inode != null) {
+ src = iip.getPath();
+ }
+ }
+ FSNamesystem fsn = fsd.getFSNamesystem();
+ final INodeFile file = fsn.checkLease(src, holder, inode, fileId);
+ Preconditions.checkState(file.isUnderConstruction());
+
+ Block localBlock = ExtendedBlock.getLocalBlock(b);
+ fsd.writeLock();
+ try {
+ // Remove the block from the pending creates list
+ if (!unprotectedRemoveBlock(fsd, src, iip, file, localBlock)) {
+ return;
+ }
+ } finally {
+ fsd.writeUnlock();
+ }
+ persistBlocks(fsd, src, file, false);
+ }
+
+ static void checkBlock(FSNamesystem fsn, ExtendedBlock block)
+ throws IOException {
+ String bpId = fsn.getBlockPoolId();
+ if (block != null && !bpId.equals(block.getBlockPoolId())) {
+ throw new IOException("Unexpected BlockPoolId " + block.getBlockPoolId()
+ + " - expected " + bpId);
+ }
+ }
+
+ /**
+ * Part I of getAdditionalBlock().
+ * Analyze the state of the file under read lock to determine if the client
+ * can add a new block, detect potential retries, lease mismatches,
+ * and minimal replication of the penultimate block.
+ *
+ * Generate target DataNode locations for the new block,
+ * but do not create the new block yet.
+ */
+ static ValidateAddBlockResult validateAddBlock(
+ FSNamesystem fsn, FSPermissionChecker pc,
+ String src, long fileId, String clientName,
+ ExtendedBlock previous, LocatedBlock[] onRetryBlock) throws IOException {
+ final long blockSize;
+ final int replication;
+ final byte storagePolicyID;
+ String clientMachine;
+
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ src = fsn.dir.resolvePath(pc, src, pathComponents);
+ FileState fileState = analyzeFileState(fsn, src, fileId, clientName,
+ previous, onRetryBlock);
+ final INodeFile pendingFile = fileState.inode;
+ // Check if the penultimate block is minimally replicated
+ if (!fsn.checkFileProgress(src, pendingFile, false)) {
+ throw new NotReplicatedYetException("Not replicated yet: " + src);
+ }
+
+ if (onRetryBlock[0] != null && onRetryBlock[0].getLocations().length > 0) {
+ // This is a retry. No need to generate new locations.
+ // Use the last block if it has locations.
+ return null;
+ }
+ if (pendingFile.getBlocks().length >= fsn.maxBlocksPerFile) {
+ throw new IOException("File has reached the limit on maximum number of"
+ + " blocks (" + DFSConfigKeys.DFS_NAMENODE_MAX_BLOCKS_PER_FILE_KEY
+ + "): " + pendingFile.getBlocks().length + " >= "
+ + fsn.maxBlocksPerFile);
+ }
+ blockSize = pendingFile.getPreferredBlockSize();
+ clientMachine = pendingFile.getFileUnderConstructionFeature()
+ .getClientMachine();
+ replication = pendingFile.getFileReplication();
+ storagePolicyID = pendingFile.getStoragePolicyID();
+ return new ValidateAddBlockResult(blockSize, replication, storagePolicyID,
+ clientMachine);
+ }
+
+ static LocatedBlock makeLocatedBlock(FSNamesystem fsn, Block blk,
+ DatanodeStorageInfo[] locs, long offset) throws IOException {
+ LocatedBlock lBlk = BlockManager.newLocatedBlock(fsn.getExtendedBlock(blk),
+ locs, offset, false);
+ fsn.getBlockManager().setBlockToken(lBlk,
+ BlockTokenIdentifier.AccessMode.WRITE);
+ return lBlk;
+ }
+
+ /**
+ * Part II of getAdditionalBlock().
+ * Should repeat the same analysis of the file state as in Part 1,
+ * but under the write lock.
+ * If the conditions still hold, then allocate a new block with
+ * the new targets, add it to the INode and to the BlocksMap.
+ */
+ static LocatedBlock storeAllocatedBlock(FSNamesystem fsn, String src,
+ long fileId, String clientName, ExtendedBlock previous,
+ DatanodeStorageInfo[] targets) throws IOException {
+ long offset;
+ // Run the full analysis again, since things could have changed
+ // while chooseTarget() was executing.
+ LocatedBlock[] onRetryBlock = new LocatedBlock[1];
+ FileState fileState = analyzeFileState(fsn, src, fileId, clientName,
+ previous, onRetryBlock);
+ final INodeFile pendingFile = fileState.inode;
+ src = fileState.path;
+
+ if (onRetryBlock[0] != null) {
+ if (onRetryBlock[0].getLocations().length > 0) {
+ // This is a retry. Just return the last block if having locations.
+ return onRetryBlock[0];
+ } else {
+ // add new chosen targets to already allocated block and return
+ BlockInfoContiguous lastBlockInFile = pendingFile.getLastBlock();
+ ((BlockInfoContiguousUnderConstruction) lastBlockInFile)
+ .setExpectedLocations(targets);
+ offset = pendingFile.computeFileSize();
+ return makeLocatedBlock(fsn, lastBlockInFile, targets, offset);
+ }
+ }
+
+ // commit the last block and complete it if it has minimum replicas
+ fsn.commitOrCompleteLastBlock(pendingFile, fileState.iip,
+ ExtendedBlock.getLocalBlock(previous));
+
+ // allocate new block, record block locations in INode.
+ Block newBlock = fsn.createNewBlock();
+ INodesInPath inodesInPath = INodesInPath.fromINode(pendingFile);
+ saveAllocatedBlock(fsn, src, inodesInPath, newBlock, targets);
+
+ persistNewBlock(fsn, src, pendingFile);
+ offset = pendingFile.computeFileSize();
+
+ // Return located block
+ return makeLocatedBlock(fsn, newBlock, targets, offset);
+ }
+
+ static DatanodeStorageInfo[] chooseTargetForNewBlock(
+ BlockManager bm, String src, DatanodeInfo[] excludedNodes, String[]
+ favoredNodes, ValidateAddBlockResult r) throws IOException {
+ Node clientNode = bm.getDatanodeManager()
+ .getDatanodeByHost(r.clientMachine);
+ if (clientNode == null) {
+ clientNode = getClientNode(bm, r.clientMachine);
+ }
+
+ Set<Node> excludedNodesSet = null;
+ if (excludedNodes != null) {
+ excludedNodesSet = new HashSet<>(excludedNodes.length);
+ Collections.addAll(excludedNodesSet, excludedNodes);
+ }
+ List<String> favoredNodesList = (favoredNodes == null) ? null
+ : Arrays.asList(favoredNodes);
+
+ // choose targets for the new block to be allocated.
+ return bm.chooseTarget4NewBlock(src, r.replication, clientNode,
+ excludedNodesSet, r.blockSize,
+ favoredNodesList, r.storagePolicyID);
+ }
+
+ /**
+ * Resolve clientmachine address to get a network location path
+ */
+ static Node getClientNode(BlockManager bm, String clientMachine) {
+ List<String> hosts = new ArrayList<>(1);
+ hosts.add(clientMachine);
+ List<String> rName = bm.getDatanodeManager()
+ .resolveNetworkLocation(hosts);
+ Node clientNode = null;
+ if (rName != null) {
+ // Able to resolve clientMachine mapping.
+ // Create a temp node to findout the rack local nodes
+ clientNode = new NodeBase(rName.get(0) + NodeBase.PATH_SEPARATOR_STR
+ + clientMachine);
+ }
+ return clientNode;
+ }
+
+ /**
+ * Add a block to the file. Returns a reference to the added block.
+ */
+ private static BlockInfoContiguous addBlock(
+ FSDirectory fsd, String path, INodesInPath inodesInPath, Block block,
+ DatanodeStorageInfo[] targets) throws IOException {
+ fsd.writeLock();
+ try {
+ final INodeFile fileINode = inodesInPath.getLastINode().asFile();
+ Preconditions.checkState(fileINode.isUnderConstruction());
+
+ // check quota limits and updated space consumed
+ fsd.updateCount(inodesInPath, 0, fileINode.getPreferredBlockSize(),
+ fileINode.getPreferredBlockReplication(), true);
+
+ // associate new last block for the file
+ BlockInfoContiguousUnderConstruction blockInfo =
+ new BlockInfoContiguousUnderConstruction(
+ block,
+ fileINode.getFileReplication(),
+ HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION,
+ targets);
+ fsd.getBlockManager().addBlockCollection(blockInfo, fileINode);
+ fileINode.addBlock(blockInfo);
+
+ if(NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("DIR* FSDirectory.addBlock: "
+ + path + " with " + block
+ + " block is added to the in-memory "
+ + "file system");
+ }
+ return blockInfo;
+ } finally {
+ fsd.writeUnlock();
+ }
+ }
+
+ private static FileState analyzeFileState(
+ FSNamesystem fsn, String src, long fileId, String clientName,
+ ExtendedBlock previous, LocatedBlock[] onRetryBlock)
+ throws IOException {
+ assert fsn.hasReadLock();
+
+ checkBlock(fsn, previous);
+ onRetryBlock[0] = null;
+ fsn.checkNameNodeSafeMode("Cannot add block to " + src);
+
+ // have we exceeded the configured limit of fs objects.
+ fsn.checkFsObjectLimit();
+
+ Block previousBlock = ExtendedBlock.getLocalBlock(previous);
+ final INode inode;
+ final INodesInPath iip;
+ if (fileId == HdfsConstants.GRANDFATHER_INODE_ID) {
+ // Older clients may not have given us an inode ID to work with.
+ // In this case, we have to try to resolve the path and hope it
+ // hasn't changed or been deleted since the file was opened for write.
+ iip = fsn.dir.getINodesInPath4Write(src);
+ inode = iip.getLastINode();
+ } else {
+ // Newer clients pass the inode ID, so we can just get the inode
+ // directly.
+ inode = fsn.dir.getInode(fileId);
+ iip = INodesInPath.fromINode(inode);
+ if (inode != null) {
+ src = iip.getPath();
+ }
+ }
+ final INodeFile file = fsn.checkLease(src, clientName,
+ inode, fileId);
+ BlockInfoContiguous lastBlockInFile = file.getLastBlock();
+ if (!Block.matchingIdAndGenStamp(previousBlock, lastBlockInFile)) {
+ // The block that the client claims is the current last block
+ // doesn't match up with what we think is the last block. There are
+ // four possibilities:
+ // 1) This is the first block allocation of an append() pipeline
+ // which started appending exactly at or exceeding the block boundary.
+ // In this case, the client isn't passed the previous block,
+ // so it makes the allocateBlock() call with previous=null.
+ // We can distinguish this since the last block of the file
+ // will be exactly a full block.
+ // 2) This is a retry from a client that missed the response of a
+ // prior getAdditionalBlock() call, perhaps because of a network
+ // timeout, or because of an HA failover. In that case, we know
+ // by the fact that the client is re-issuing the RPC that it
+ // never began to write to the old block. Hence it is safe to
+ // to return the existing block.
+ // 3) This is an entirely bogus request/bug -- we should error out
+ // rather than potentially appending a new block with an empty
+ // one in the middle, etc
+ // 4) This is a retry from a client that timed out while
+ // the prior getAdditionalBlock() is still being processed,
+ // currently working on chooseTarget().
+ // There are no means to distinguish between the first and
+ // the second attempts in Part I, because the first one hasn't
+ // changed the namesystem state yet.
+ // We run this analysis again in Part II where case 4 is impossible.
+
+ BlockInfoContiguous penultimateBlock = file.getPenultimateBlock();
+ if (previous == null &&
+ lastBlockInFile != null &&
+ lastBlockInFile.getNumBytes() >= file.getPreferredBlockSize() &&
+ lastBlockInFile.isComplete()) {
+ // Case 1
+ if (NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug(
+ "BLOCK* NameSystem.allocateBlock: handling block allocation" +
+ " writing to a file with a complete previous block: src=" +
+ src + " lastBlock=" + lastBlockInFile);
+ }
+ } else if (Block.matchingIdAndGenStamp(penultimateBlock, previousBlock)) {
+ if (lastBlockInFile.getNumBytes() != 0) {
+ throw new IOException(
+ "Request looked like a retry to allocate block " +
+ lastBlockInFile + " but it already contains " +
+ lastBlockInFile.getNumBytes() + " bytes");
+ }
+
+ // Case 2
+ // Return the last block.
+ NameNode.stateChangeLog.info("BLOCK* allocateBlock: caught retry for " +
+ "allocation of a new block in " + src + ". Returning previously" +
+ " allocated block " + lastBlockInFile);
+ long offset = file.computeFileSize();
+ BlockInfoContiguousUnderConstruction lastBlockUC =
+ (BlockInfoContiguousUnderConstruction) lastBlockInFile;
+ onRetryBlock[0] = makeLocatedBlock(fsn, lastBlockInFile,
+ lastBlockUC.getExpectedStorageLocations(), offset);
+ return new FileState(file, src, iip);
+ } else {
+ // Case 3
+ throw new IOException("Cannot allocate block in " + src + ": " +
+ "passed 'previous' block " + previous + " does not match actual " +
+ "last block in file " + lastBlockInFile);
+ }
+ }
+ return new FileState(file, src, iip);
+ }
+
+ static boolean completeFile(FSNamesystem fsn, FSPermissionChecker pc,
+ final String srcArg, String holder, ExtendedBlock last, long fileId)
+ throws IOException {
+ String src = srcArg;
+ if (NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("DIR* NameSystem.completeFile: " +
+ src + " for " + holder);
+ }
+ checkBlock(fsn, last);
+ byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ src = fsn.dir.resolvePath(pc, src, pathComponents);
+ boolean success = completeFileInternal(fsn, src, holder,
+ ExtendedBlock.getLocalBlock(last),
+ fileId);
+ if (success) {
+ NameNode.stateChangeLog.info("DIR* completeFile: " + srcArg
+ + " is closed by " + holder);
+ }
+ return success;
+ }
+
+ private static boolean completeFileInternal(
+ FSNamesystem fsn, String src, String holder, Block last, long fileId)
+ throws IOException {
+ assert fsn.hasWriteLock();
+ final INodeFile pendingFile;
+ final INodesInPath iip;
+ INode inode = null;
+ try {
+ if (fileId == HdfsConstants.GRANDFATHER_INODE_ID) {
+ // Older clients may not have given us an inode ID to work with.
+ // In this case, we have to try to resolve the path and hope it
+ // hasn't changed or been deleted since the file was opened for write.
+ iip = fsn.dir.getINodesInPath(src, true);
+ inode = iip.getLastINode();
+ } else {
+ inode = fsn.dir.getInode(fileId);
+ iip = INodesInPath.fromINode(inode);
+ if (inode != null) {
+ src = iip.getPath();
+ }
+ }
+ pendingFile = fsn.checkLease(src, holder, inode, fileId);
+ } catch (LeaseExpiredException lee) {
+ if (inode != null && inode.isFile() &&
+ !inode.asFile().isUnderConstruction()) {
+ // This could be a retry RPC - i.e the client tried to close
+ // the file, but missed the RPC response. Thus, it is trying
+ // again to close the file. If the file still exists and
+ // the client's view of the last block matches the actual
+ // last block, then we'll treat it as a successful close.
+ // See HDFS-3031.
+ final Block realLastBlock = inode.asFile().getLastBlock();
+ if (Block.matchingIdAndGenStamp(last, realLastBlock)) {
+ NameNode.stateChangeLog.info("DIR* completeFile: " +
+ "request from " + holder + " to complete inode " + fileId +
+ "(" + src + ") which is already closed. But, it appears to be " +
+ "an RPC retry. Returning success");
+ return true;
+ }
+ }
+ throw lee;
+ }
+ // Check the state of the penultimate block. It should be completed
+ // before attempting to complete the last one.
+ if (!fsn.checkFileProgress(src, pendingFile, false)) {
+ return false;
+ }
+
+ // commit the last block and complete it if it has minimum replicas
+ fsn.commitOrCompleteLastBlock(pendingFile, iip, last);
+
+ if (!fsn.checkFileProgress(src, pendingFile, true)) {
+ return false;
+ }
+
+ fsn.finalizeINodeFileUnderConstruction(src, pendingFile,
+ Snapshot.CURRENT_STATE_ID);
+ return true;
+ }
+
+ /**
+ * Persist the new block (the last block of the given file).
+ */
+ private static void persistNewBlock(
+ FSNamesystem fsn, String path, INodeFile file) {
+ Preconditions.checkArgument(file.isUnderConstruction());
+ fsn.getEditLog().logAddBlock(path, file);
+ if (NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("persistNewBlock: "
+ + path + " with new block " + file.getLastBlock().toString()
+ + ", current total block count is " + file.getBlocks().length);
+ }
+ }
+
+ /**
+ * Save allocated block at the given pending filename
+ *
+ * @param fsn FSNamesystem
+ * @param src path to the file
+ * @param inodesInPath representing each of the components of src.
+ * The last INode is the INode for {@code src} file.
+ * @param newBlock newly allocated block to be save
+ * @param targets target datanodes where replicas of the new block is placed
+ * @throws QuotaExceededException If addition of block exceeds space quota
+ */
+ private static void saveAllocatedBlock(
+ FSNamesystem fsn, String src, INodesInPath inodesInPath, Block newBlock,
+ DatanodeStorageInfo[] targets)
+ throws IOException {
+ assert fsn.hasWriteLock();
+ BlockInfoContiguous b = addBlock(fsn.dir, src, inodesInPath, newBlock,
+ targets);
+ NameNode.stateChangeLog.info("BLOCK* allocate " + b + " for " + src);
+ DatanodeStorageInfo.incrementBlocksScheduled(targets);
+ }
+
+ private static class FileState {
+ final INodeFile inode;
+ final String path;
+ final INodesInPath iip;
+
+ FileState(INodeFile inode, String fullPath, INodesInPath iip) {
+ this.inode = inode;
+ this.path = fullPath;
+ this.iip = iip;
+ }
+ }
+
+ static class ValidateAddBlockResult {
+ final long blockSize;
+ final int replication;
+ final byte storagePolicyID;
+ final String clientMachine;
+
+ ValidateAddBlockResult(
+ long blockSize, int replication, byte storagePolicyID,
+ String clientMachine) {
+ this.blockSize = blockSize;
+ this.replication = replication;
+ this.storagePolicyID = storagePolicyID;
+ this.clientMachine = clientMachine;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/480b65c7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 1583815..c2ed956 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -55,12 +55,9 @@ import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
-import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.util.ByteArray;
import org.apache.hadoop.hdfs.util.EnumCounters;
@@ -308,7 +305,7 @@ public class FSDirectory implements Closeable {
return namesystem;
}
- private BlockManager getBlockManager() {
+ BlockManager getBlockManager() {
return getFSNamesystem().getBlockManager();
}
@@ -479,79 +476,6 @@ public class FSDirectory implements Closeable {
}
/**
- * Add a block to the file. Returns a reference to the added block.
- */
- BlockInfoContiguous addBlock(String path, INodesInPath inodesInPath,
- Block block, DatanodeStorageInfo[] targets) throws IOException {
- writeLock();
- try {
- final INodeFile fileINode = inodesInPath.getLastINode().asFile();
- Preconditions.checkState(fileINode.isUnderConstruction());
-
- // check quota limits and updated space consumed
- updateCount(inodesInPath, 0, fileINode.getPreferredBlockSize(),
- fileINode.getPreferredBlockReplication(), true);
-
- // associate new last block for the file
- BlockInfoContiguousUnderConstruction blockInfo =
- new BlockInfoContiguousUnderConstruction(
- block,
- fileINode.getFileReplication(),
- BlockUCState.UNDER_CONSTRUCTION,
- targets);
- getBlockManager().addBlockCollection(blockInfo, fileINode);
- fileINode.addBlock(blockInfo);
-
- if(NameNode.stateChangeLog.isDebugEnabled()) {
- NameNode.stateChangeLog.debug("DIR* FSDirectory.addBlock: "
- + path + " with " + block
- + " block is added to the in-memory "
- + "file system");
- }
- return blockInfo;
- } finally {
- writeUnlock();
- }
- }
-
- /**
- * Remove a block from the file.
- * @return Whether the block exists in the corresponding file
- */
- boolean removeBlock(String path, INodesInPath iip, INodeFile fileNode,
- Block block) throws IOException {
- Preconditions.checkArgument(fileNode.isUnderConstruction());
- writeLock();
- try {
- return unprotectedRemoveBlock(path, iip, fileNode, block);
- } finally {
- writeUnlock();
- }
- }
-
- boolean unprotectedRemoveBlock(String path, INodesInPath iip,
- INodeFile fileNode, Block block) throws IOException {
- // modify file-> block and blocksMap
- // fileNode should be under construction
- BlockInfoContiguousUnderConstruction uc = fileNode.removeLastBlock(block);
- if (uc == null) {
- return false;
- }
- getBlockManager().removeBlockFromMap(block);
-
- if(NameNode.stateChangeLog.isDebugEnabled()) {
- NameNode.stateChangeLog.debug("DIR* FSDirectory.removeBlock: "
- +path+" with "+block
- +" block is removed from the file system");
- }
-
- // update space consumed
- updateCount(iip, 0, -fileNode.getPreferredBlockSize(),
- fileNode.getPreferredBlockReplication(), true);
- return true;
- }
-
- /**
* This is a wrapper for resolvePath(). If the path passed
* is prefixed with /.reserved/raw, then it checks to ensure that the caller
* has super user privileges.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/480b65c7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index f75c117..dec1298 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -1037,7 +1037,8 @@ public class FSEditLogLoader {
+ path);
}
Block oldBlock = oldBlocks[oldBlocks.length - 1];
- boolean removed = fsDir.unprotectedRemoveBlock(path, iip, file, oldBlock);
+ boolean removed = FSDirWriteFileOp.unprotectedRemoveBlock(
+ fsDir, path, iip, file, oldBlock);
if (!removed && !(op instanceof UpdateBlocksOp)) {
throw new IOException("Trying to delete non-existant block " + oldBlock);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/480b65c7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 4d82fab..0fec5ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -268,7 +268,6 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node;
-import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
@@ -484,7 +483,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
private final long maxFsObjects; // maximum number of fs objects
private final long minBlockSize; // minimum block size
- private final long maxBlocksPerFile; // maximum # of blocks per file
+ final long maxBlocksPerFile; // maximum # of blocks per file
// precision of access times.
private final long accessTimePrecision;
@@ -614,7 +613,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
boolean isHaEnabled() {
return haEnabled;
}
-
+
/**
* Check the supplied configuration for correctness.
* @param conf Supplies the configuration to validate.
@@ -1863,8 +1862,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
: dir.getFileEncryptionInfo(inode, iip.getPathSnapshotId(), iip);
final LocatedBlocks blocks = blockManager.createLocatedBlocks(
- inode.getBlocks(iip.getPathSnapshotId()), fileSize,
- isUc, offset, length, needBlockToken, iip.isSnapshot(), feInfo);
+ inode.getBlocks(iip.getPathSnapshotId()), fileSize, isUc, offset,
+ length, needBlockToken, iip.isSnapshot(), feInfo);
// Set caching information for the located blocks.
for (LocatedBlock lb : blocks.getLocatedBlocks()) {
@@ -2232,8 +2231,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot set storage policy for " + src);
- auditStat = FSDirAttrOp.setStoragePolicy(
- dir, blockManager, src, policyName);
+ auditStat = FSDirAttrOp.setStoragePolicy(dir, blockManager, src,
+ policyName);
} catch (AccessControlException e) {
logAuditEvent(false, "setStoragePolicy", src);
throw e;
@@ -2621,7 +2620,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return toRemoveBlocks;
} catch (IOException ie) {
NameNode.stateChangeLog.warn("DIR* NameSystem.startFile: " + src + " " +
- ie.getMessage());
+ ie.getMessage());
throw ie;
}
}
@@ -2703,8 +2702,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
"Cannot append to lazy persist file " + src);
}
// Opening an existing file for append - may need to recover lease.
- recoverLeaseInternal(RecoverLeaseOp.APPEND_FILE,
- iip, src, holder, clientMachine, false);
+ recoverLeaseInternal(RecoverLeaseOp.APPEND_FILE, iip, src, holder,
+ clientMachine, false);
final BlockInfoContiguous lastBlock = myFile.getLastBlock();
// Check that the block has at least minimum replication.
@@ -3042,290 +3041,49 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* are replicated. Will return an empty 2-elt array if we want the
* client to "try again later".
*/
- LocatedBlock getAdditionalBlock(String src, long fileId, String clientName,
- ExtendedBlock previous, Set<Node> excludedNodes,
- List<String> favoredNodes) throws IOException {
- LocatedBlock[] onRetryBlock = new LocatedBlock[1];
- DatanodeStorageInfo targets[] = getNewBlockTargets(src, fileId,
- clientName, previous, excludedNodes, favoredNodes, onRetryBlock);
- if (targets == null) {
- assert onRetryBlock[0] != null : "Retry block is null";
- // This is a retry. Just return the last block.
- return onRetryBlock[0];
- }
- LocatedBlock newBlock = storeAllocatedBlock(
- src, fileId, clientName, previous, targets);
- return newBlock;
- }
-
- /**
- * Part I of getAdditionalBlock().
- * Analyze the state of the file under read lock to determine if the client
- * can add a new block, detect potential retries, lease mismatches,
- * and minimal replication of the penultimate block.
- *
- * Generate target DataNode locations for the new block,
- * but do not create the new block yet.
- */
- DatanodeStorageInfo[] getNewBlockTargets(String src, long fileId,
- String clientName, ExtendedBlock previous, Set<Node> excludedNodes,
- List<String> favoredNodes, LocatedBlock[] onRetryBlock) throws IOException {
- final long blockSize;
- final int replication;
- final byte storagePolicyID;
- Node clientNode = null;
- String clientMachine = null;
-
+ LocatedBlock getAdditionalBlock(
+ String src, long fileId, String clientName, ExtendedBlock previous,
+ DatanodeInfo[] excludedNodes, String[] favoredNodes) throws IOException {
if(NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("BLOCK* getAdditionalBlock: "
+ src + " inodeId " + fileId + " for " + clientName);
}
- checkOperation(OperationCategory.READ);
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ waitForLoadingFSImage();
+ LocatedBlock[] onRetryBlock = new LocatedBlock[1];
+ FSDirWriteFileOp.ValidateAddBlockResult r;
FSPermissionChecker pc = getPermissionChecker();
+ checkOperation(OperationCategory.READ);
readLock();
try {
checkOperation(OperationCategory.READ);
- src = dir.resolvePath(pc, src, pathComponents);
- FileState fileState = analyzeFileState(
- src, fileId, clientName, previous, onRetryBlock);
- final INodeFile pendingFile = fileState.inode;
- // Check if the penultimate block is minimally replicated
- if (!checkFileProgress(src, pendingFile, false)) {
- throw new NotReplicatedYetException("Not replicated yet: " + src);
- }
- src = fileState.path;
-
- if (onRetryBlock[0] != null && onRetryBlock[0].getLocations().length > 0) {
- // This is a retry. No need to generate new locations.
- // Use the last block if it has locations.
- return null;
- }
- if (pendingFile.getBlocks().length >= maxBlocksPerFile) {
- throw new IOException("File has reached the limit on maximum number of"
- + " blocks (" + DFSConfigKeys.DFS_NAMENODE_MAX_BLOCKS_PER_FILE_KEY
- + "): " + pendingFile.getBlocks().length + " >= "
- + maxBlocksPerFile);
- }
- blockSize = pendingFile.getPreferredBlockSize();
- clientMachine = pendingFile.getFileUnderConstructionFeature()
- .getClientMachine();
- clientNode = blockManager.getDatanodeManager().getDatanodeByHost(
- clientMachine);
- replication = pendingFile.getFileReplication();
- storagePolicyID = pendingFile.getStoragePolicyID();
+ r = FSDirWriteFileOp.validateAddBlock(this, pc, src, fileId, clientName,
+ previous, onRetryBlock);
} finally {
readUnlock();
}
- if (clientNode == null) {
- clientNode = getClientNode(clientMachine);
+ if (r == null) {
+ assert onRetryBlock[0] != null : "Retry block is null";
+ // This is a retry. Just return the last block.
+ return onRetryBlock[0];
}
- // choose targets for the new block to be allocated.
- return getBlockManager().chooseTarget4NewBlock(
- src, replication, clientNode, excludedNodes, blockSize, favoredNodes,
- storagePolicyID);
- }
+ DatanodeStorageInfo[] targets = FSDirWriteFileOp.chooseTargetForNewBlock(
+ blockManager, src, excludedNodes, favoredNodes, r);
- /**
- * Part II of getAdditionalBlock().
- * Should repeat the same analysis of the file state as in Part 1,
- * but under the write lock.
- * If the conditions still hold, then allocate a new block with
- * the new targets, add it to the INode and to the BlocksMap.
- */
- LocatedBlock storeAllocatedBlock(String src, long fileId, String clientName,
- ExtendedBlock previous, DatanodeStorageInfo[] targets) throws IOException {
- Block newBlock = null;
- long offset;
checkOperation(OperationCategory.WRITE);
- waitForLoadingFSImage();
writeLock();
+ LocatedBlock lb;
try {
checkOperation(OperationCategory.WRITE);
- // Run the full analysis again, since things could have changed
- // while chooseTarget() was executing.
- LocatedBlock[] onRetryBlock = new LocatedBlock[1];
- FileState fileState =
- analyzeFileState(src, fileId, clientName, previous, onRetryBlock);
- final INodeFile pendingFile = fileState.inode;
- src = fileState.path;
-
- if (onRetryBlock[0] != null) {
- if (onRetryBlock[0].getLocations().length > 0) {
- // This is a retry. Just return the last block if having locations.
- return onRetryBlock[0];
- } else {
- // add new chosen targets to already allocated block and return
- BlockInfoContiguous lastBlockInFile = pendingFile.getLastBlock();
- ((BlockInfoContiguousUnderConstruction) lastBlockInFile)
- .setExpectedLocations(targets);
- offset = pendingFile.computeFileSize();
- return makeLocatedBlock(lastBlockInFile, targets, offset);
- }
- }
-
- // commit the last block and complete it if it has minimum replicas
- commitOrCompleteLastBlock(pendingFile, fileState.iip,
- ExtendedBlock.getLocalBlock(previous));
-
- // allocate new block, record block locations in INode.
- newBlock = createNewBlock();
- INodesInPath inodesInPath = INodesInPath.fromINode(pendingFile);
- saveAllocatedBlock(src, inodesInPath, newBlock, targets);
-
- persistNewBlock(src, pendingFile);
- offset = pendingFile.computeFileSize();
+ lb = FSDirWriteFileOp.storeAllocatedBlock(
+ this, src, fileId, clientName, previous, targets);
} finally {
writeUnlock();
}
getEditLog().logSync();
-
- // Return located block
- return makeLocatedBlock(newBlock, targets, offset);
- }
-
- /*
- * Resolve clientmachine address to get a network location path
- */
- private Node getClientNode(String clientMachine) {
- List<String> hosts = new ArrayList<String>(1);
- hosts.add(clientMachine);
- List<String> rName = getBlockManager().getDatanodeManager()
- .resolveNetworkLocation(hosts);
- Node clientNode = null;
- if (rName != null) {
- // Able to resolve clientMachine mapping.
- // Create a temp node to findout the rack local nodes
- clientNode = new NodeBase(rName.get(0) + NodeBase.PATH_SEPARATOR_STR
- + clientMachine);
- }
- return clientNode;
- }
-
- static class FileState {
- public final INodeFile inode;
- public final String path;
- public final INodesInPath iip;
-
- public FileState(INodeFile inode, String fullPath, INodesInPath iip) {
- this.inode = inode;
- this.path = fullPath;
- this.iip = iip;
- }
- }
-
- FileState analyzeFileState(String src,
- long fileId,
- String clientName,
- ExtendedBlock previous,
- LocatedBlock[] onRetryBlock)
- throws IOException {
- assert hasReadLock();
-
- checkBlock(previous);
- onRetryBlock[0] = null;
- checkNameNodeSafeMode("Cannot add block to " + src);
-
- // have we exceeded the configured limit of fs objects.
- checkFsObjectLimit();
-
- Block previousBlock = ExtendedBlock.getLocalBlock(previous);
- final INode inode;
- final INodesInPath iip;
- if (fileId == HdfsConstants.GRANDFATHER_INODE_ID) {
- // Older clients may not have given us an inode ID to work with.
- // In this case, we have to try to resolve the path and hope it
- // hasn't changed or been deleted since the file was opened for write.
- iip = dir.getINodesInPath4Write(src);
- inode = iip.getLastINode();
- } else {
- // Newer clients pass the inode ID, so we can just get the inode
- // directly.
- inode = dir.getInode(fileId);
- iip = INodesInPath.fromINode(inode);
- if (inode != null) {
- src = iip.getPath();
- }
- }
- final INodeFile pendingFile = checkLease(src, clientName, inode, fileId);
- BlockInfoContiguous lastBlockInFile = pendingFile.getLastBlock();
- if (!Block.matchingIdAndGenStamp(previousBlock, lastBlockInFile)) {
- // The block that the client claims is the current last block
- // doesn't match up with what we think is the last block. There are
- // four possibilities:
- // 1) This is the first block allocation of an append() pipeline
- // which started appending exactly at or exceeding the block boundary.
- // In this case, the client isn't passed the previous block,
- // so it makes the allocateBlock() call with previous=null.
- // We can distinguish this since the last block of the file
- // will be exactly a full block.
- // 2) This is a retry from a client that missed the response of a
- // prior getAdditionalBlock() call, perhaps because of a network
- // timeout, or because of an HA failover. In that case, we know
- // by the fact that the client is re-issuing the RPC that it
- // never began to write to the old block. Hence it is safe to
- // to return the existing block.
- // 3) This is an entirely bogus request/bug -- we should error out
- // rather than potentially appending a new block with an empty
- // one in the middle, etc
- // 4) This is a retry from a client that timed out while
- // the prior getAdditionalBlock() is still being processed,
- // currently working on chooseTarget().
- // There are no means to distinguish between the first and
- // the second attempts in Part I, because the first one hasn't
- // changed the namesystem state yet.
- // We run this analysis again in Part II where case 4 is impossible.
-
- BlockInfoContiguous penultimateBlock = pendingFile.getPenultimateBlock();
- if (previous == null &&
- lastBlockInFile != null &&
- lastBlockInFile.getNumBytes() >= pendingFile.getPreferredBlockSize() &&
- lastBlockInFile.isComplete()) {
- // Case 1
- if (NameNode.stateChangeLog.isDebugEnabled()) {
- NameNode.stateChangeLog.debug(
- "BLOCK* NameSystem.allocateBlock: handling block allocation" +
- " writing to a file with a complete previous block: src=" +
- src + " lastBlock=" + lastBlockInFile);
- }
- } else if (Block.matchingIdAndGenStamp(penultimateBlock, previousBlock)) {
- if (lastBlockInFile.getNumBytes() != 0) {
- throw new IOException(
- "Request looked like a retry to allocate block " +
- lastBlockInFile + " but it already contains " +
- lastBlockInFile.getNumBytes() + " bytes");
- }
-
- // Case 2
- // Return the last block.
- NameNode.stateChangeLog.info("BLOCK* allocateBlock: " +
- "caught retry for allocation of a new block in " +
- src + ". Returning previously allocated block " + lastBlockInFile);
- long offset = pendingFile.computeFileSize();
- onRetryBlock[0] = makeLocatedBlock(lastBlockInFile,
- ((BlockInfoContiguousUnderConstruction)lastBlockInFile).getExpectedStorageLocations(),
- offset);
- return new FileState(pendingFile, src, iip);
- } else {
- // Case 3
- throw new IOException("Cannot allocate block in " + src + ": " +
- "passed 'previous' block " + previous + " does not match actual " +
- "last block in file " + lastBlockInFile);
- }
- }
- return new FileState(pendingFile, src, iip);
- }
-
- LocatedBlock makeLocatedBlock(Block blk, DatanodeStorageInfo[] locs,
- long offset) throws IOException {
- LocatedBlock lBlk = BlockManager.newLocatedBlock(
- getExtendedBlock(blk), locs, offset, false);
- getBlockManager().setBlockToken(
- lBlk, BlockTokenIdentifier.AccessMode.WRITE);
- return lBlk;
+ return lb;
}
/** @see ClientProtocol#getAdditionalDatanode */
@@ -3378,7 +3136,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
if (clientnode == null) {
- clientnode = getClientNode(clientMachine);
+ clientnode = FSDirWriteFileOp.getClientNode(blockManager, clientMachine);
}
// choose new datanodes.
@@ -3394,60 +3152,32 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
/**
* The client would like to let go of the given block
*/
- boolean abandonBlock(ExtendedBlock b, long fileId, String src, String holder)
+ void abandonBlock(ExtendedBlock b, long fileId, String src, String holder)
throws IOException {
if(NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("BLOCK* NameSystem.abandonBlock: " + b
+ "of file " + src);
}
+ waitForLoadingFSImage();
checkOperation(OperationCategory.WRITE);
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
FSPermissionChecker pc = getPermissionChecker();
- waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot abandon block " + b + " for file" + src);
- src = dir.resolvePath(pc, src, pathComponents);
-
- final INode inode;
- final INodesInPath iip;
- if (fileId == HdfsConstants.GRANDFATHER_INODE_ID) {
- // Older clients may not have given us an inode ID to work with.
- // In this case, we have to try to resolve the path and hope it
- // hasn't changed or been deleted since the file was opened for write.
- iip = dir.getINodesInPath(src, true);
- inode = iip.getLastINode();
- } else {
- inode = dir.getInode(fileId);
- iip = INodesInPath.fromINode(inode);
- if (inode != null) {
- src = iip.getPath();
- }
- }
- final INodeFile file = checkLease(src, holder, inode, fileId);
-
- // Remove the block from the pending creates list
- boolean removed = dir.removeBlock(src, iip, file,
- ExtendedBlock.getLocalBlock(b));
- if (!removed) {
- return true;
- }
+ FSDirWriteFileOp.abandonBlock(dir, pc, b, fileId, src, holder);
if(NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("BLOCK* NameSystem.abandonBlock: "
+ b + " is removed from pendingCreates");
}
- persistBlocks(src, file, false);
} finally {
writeUnlock();
}
getEditLog().logSync();
-
- return true;
}
- private INodeFile checkLease(String src, String holder, INode inode,
- long fileId) throws LeaseExpiredException, FileNotFoundException {
+ INodeFile checkLease(
+ String src, String holder, INode inode, long fileId) throws LeaseExpiredException, FileNotFoundException {
assert hasReadLock();
final String ident = src + " (inode " + fileId + ")";
if (inode == null) {
@@ -3492,120 +3222,30 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* (e.g if not all blocks have reached minimum replication yet)
* @throws IOException on error (eg lease mismatch, file not open, file deleted)
*/
- boolean completeFile(final String srcArg, String holder,
+ boolean completeFile(final String src, String holder,
ExtendedBlock last, long fileId)
- throws SafeModeException, UnresolvedLinkException, IOException {
- String src = srcArg;
- if (NameNode.stateChangeLog.isDebugEnabled()) {
- NameNode.stateChangeLog.debug("DIR* NameSystem.completeFile: " +
- src + " for " + holder);
- }
- checkBlock(last);
+ throws IOException {
boolean success = false;
checkOperation(OperationCategory.WRITE);
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
- FSPermissionChecker pc = getPermissionChecker();
waitForLoadingFSImage();
+ FSPermissionChecker pc = getPermissionChecker();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot complete file " + src);
- src = dir.resolvePath(pc, src, pathComponents);
- success = completeFileInternal(src, holder,
- ExtendedBlock.getLocalBlock(last), fileId);
+ success = FSDirWriteFileOp.completeFile(this, pc, src, holder, last,
+ fileId);
} finally {
writeUnlock();
}
getEditLog().logSync();
- if (success) {
- NameNode.stateChangeLog.info("DIR* completeFile: " + srcArg
- + " is closed by " + holder);
- }
return success;
}
- private boolean completeFileInternal(String src, String holder, Block last,
- long fileId) throws IOException {
- assert hasWriteLock();
- final INodeFile pendingFile;
- final INodesInPath iip;
- INode inode = null;
- try {
- if (fileId == HdfsConstants.GRANDFATHER_INODE_ID) {
- // Older clients may not have given us an inode ID to work with.
- // In this case, we have to try to resolve the path and hope it
- // hasn't changed or been deleted since the file was opened for write.
- iip = dir.getINodesInPath(src, true);
- inode = iip.getLastINode();
- } else {
- inode = dir.getInode(fileId);
- iip = INodesInPath.fromINode(inode);
- if (inode != null) {
- src = iip.getPath();
- }
- }
- pendingFile = checkLease(src, holder, inode, fileId);
- } catch (LeaseExpiredException lee) {
- if (inode != null && inode.isFile() &&
- !inode.asFile().isUnderConstruction()) {
- // This could be a retry RPC - i.e the client tried to close
- // the file, but missed the RPC response. Thus, it is trying
- // again to close the file. If the file still exists and
- // the client's view of the last block matches the actual
- // last block, then we'll treat it as a successful close.
- // See HDFS-3031.
- final Block realLastBlock = inode.asFile().getLastBlock();
- if (Block.matchingIdAndGenStamp(last, realLastBlock)) {
- NameNode.stateChangeLog.info("DIR* completeFile: " +
- "request from " + holder + " to complete inode " + fileId +
- "(" + src + ") which is already closed. But, it appears to be " +
- "an RPC retry. Returning success");
- return true;
- }
- }
- throw lee;
- }
- // Check the state of the penultimate block. It should be completed
- // before attempting to complete the last one.
- if (!checkFileProgress(src, pendingFile, false)) {
- return false;
- }
-
- // commit the last block and complete it if it has minimum replicas
- commitOrCompleteLastBlock(pendingFile, iip, last);
-
- if (!checkFileProgress(src, pendingFile, true)) {
- return false;
- }
-
- finalizeINodeFileUnderConstruction(src, pendingFile,
- Snapshot.CURRENT_STATE_ID);
- return true;
- }
-
- /**
- * Save allocated block at the given pending filename
- *
- * @param src path to the file
- * @param inodesInPath representing each of the components of src.
- * The last INode is the INode for {@code src} file.
- * @param newBlock newly allocated block to be save
- * @param targets target datanodes where replicas of the new block is placed
- * @throws QuotaExceededException If addition of block exceeds space quota
- */
- private void saveAllocatedBlock(String src, INodesInPath inodesInPath,
- Block newBlock, DatanodeStorageInfo[] targets)
- throws IOException {
- assert hasWriteLock();
- BlockInfoContiguous b = dir.addBlock(src, inodesInPath, newBlock, targets);
- NameNode.stateChangeLog.info("BLOCK* allocate " + b + " for " + src);
- DatanodeStorageInfo.incrementBlocksScheduled(targets);
- }
-
/**
* Create new block with a unique block id and a new generation stamp.
*/
- private Block createNewBlock() throws IOException {
+ Block createNewBlock() throws IOException {
assert hasWriteLock();
Block b = new Block(nextBlockId(), 0, 0);
// Increment the generation stamp for every new block.
@@ -3997,7 +3637,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
pendingFile.getFileUnderConstructionFeature().updateLengthOfLastBlock(
pendingFile, lastBlockLength);
}
- persistBlocks(src, pendingFile, false);
+ FSDirWriteFileOp.persistBlocks(dir, src, pendingFile, false);
} finally {
writeUnlock();
}
@@ -4167,8 +3807,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return leaseManager.reassignLease(lease, pendingFile, newHolder);
}
- private void commitOrCompleteLastBlock(final INodeFile fileINode,
- final INodesInPath iip, final Block commitBlock) throws IOException {
+ void commitOrCompleteLastBlock(
+ final INodeFile fileINode, final INodesInPath iip,
+ final Block commitBlock) throws IOException {
assert hasWriteLock();
Preconditions.checkArgument(fileINode.isUnderConstruction());
if (!blockManager.commitOrCompleteLastBlock(fileINode, commitBlock)) {
@@ -4186,14 +3827,14 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
}
- private void finalizeINodeFileUnderConstruction(String src,
- INodeFile pendingFile, int latestSnapshot) throws IOException {
+ void finalizeINodeFileUnderConstruction(
+ String src, INodeFile pendingFile, int latestSnapshot) throws IOException {
assert hasWriteLock();
FileUnderConstructionFeature uc = pendingFile.getFileUnderConstructionFeature();
Preconditions.checkArgument(uc != null);
leaseManager.removeLease(uc.getClientName(), pendingFile);
-
+
pendingFile.recordModification(latestSnapshot);
// The file is no longer pending.
@@ -4405,7 +4046,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
} else {
// If this commit does not want to close the file, persist blocks
src = iFile.getFullPathName();
- persistBlocks(src, iFile, false);
+ FSDirWriteFileOp.persistBlocks(dir, src, iFile, false);
}
} finally {
writeUnlock();
@@ -4596,24 +4237,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
/**
- * Persist the block list for the inode.
- * @param path
- * @param file
- * @param logRetryCache
- */
- private void persistBlocks(String path, INodeFile file,
- boolean logRetryCache) {
- assert hasWriteLock();
- Preconditions.checkArgument(file.isUnderConstruction());
- getEditLog().logUpdateBlocks(path, file, logRetryCache);
- if(NameNode.stateChangeLog.isDebugEnabled()) {
- NameNode.stateChangeLog.debug("persistBlocks: " + path
- + " with " + file.getBlocks().length + " blocks is persisted to" +
- " the file system");
- }
- }
-
- /**
* Close file.
* @param path
* @param file
@@ -4800,13 +4423,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
public FSEditLog getEditLog() {
return getFSImage().getEditLog();
- }
-
- private void checkBlock(ExtendedBlock block) throws IOException {
- if (block != null && !this.blockPoolId.equals(block.getBlockPoolId())) {
- throw new IOException("Unexpected BlockPoolId " + block.getBlockPoolId()
- + " - expected " + blockPoolId);
- }
}
@Metric({"MissingBlocks", "Number of missing blocks"})
@@ -5080,21 +4696,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
/**
- * Persist the new block (the last block of the given file).
- * @param path
- * @param file
- */
- private void persistNewBlock(String path, INodeFile file) {
- Preconditions.checkArgument(file.isUnderConstruction());
- getEditLog().logAddBlock(path, file);
- if (NameNode.stateChangeLog.isDebugEnabled()) {
- NameNode.stateChangeLog.debug("persistNewBlock: "
- + path + " with new block " + file.getLastBlock().toString()
- + ", current total block count is " + file.getBlocks().length);
- }
- }
-
- /**
* SafeModeInfo contains information related to the safe mode.
* <p>
* An instance of {@link SafeModeInfo} is created when the name node
@@ -6399,7 +6000,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
blockinfo.setExpectedLocations(storages);
String src = pendingFile.getFullPathName();
- persistBlocks(src, pendingFile, logRetryCache);
+ FSDirWriteFileOp.persistBlocks(dir, src, pendingFile, logRetryCache);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/480b65c7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 3311609..0d416a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -713,23 +713,11 @@ class NameNodeRpcServer implements NamenodeProtocols {
String[] favoredNodes)
throws IOException {
checkNNStartup();
- if (stateChangeLog.isDebugEnabled()) {
- stateChangeLog.debug("*BLOCK* NameNode.addBlock: file " + src
- + " fileId=" + fileId + " for " + clientName);
- }
- Set<Node> excludedNodesSet = null;
- if (excludedNodes != null) {
- excludedNodesSet = new HashSet<Node>(excludedNodes.length);
- for (Node node : excludedNodes) {
- excludedNodesSet.add(node);
- }
- }
- List<String> favoredNodesList = (favoredNodes == null) ? null
- : Arrays.asList(favoredNodes);
LocatedBlock locatedBlock = namesystem.getAdditionalBlock(src, fileId,
- clientName, previous, excludedNodesSet, favoredNodesList);
- if (locatedBlock != null)
+ clientName, previous, excludedNodes, favoredNodes);
+ if (locatedBlock != null) {
metrics.incrAddBlockOps();
+ }
return locatedBlock;
}
@@ -770,13 +758,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
public void abandonBlock(ExtendedBlock b, long fileId, String src,
String holder) throws IOException {
checkNNStartup();
- if(stateChangeLog.isDebugEnabled()) {
- stateChangeLog.debug("*BLOCK* NameNode.abandonBlock: "
- +b+" of file "+src);
- }
- if (!namesystem.abandonBlock(b, fileId, src, holder)) {
- throw new IOException("Cannot abandon block during write to " + src);
- }
+ namesystem.abandonBlock(b, fileId, src, holder);
}
@Override // ClientProtocol
@@ -784,10 +766,6 @@ class NameNodeRpcServer implements NamenodeProtocols {
ExtendedBlock last, long fileId)
throws IOException {
checkNNStartup();
- if(stateChangeLog.isDebugEnabled()) {
- stateChangeLog.debug("*DIR* NameNode.complete: "
- + src + " fileId=" + fileId +" for " + clientName);
- }
return namesystem.completeFile(src, clientName, last, fileId);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/480b65c7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java
index 5a4134c..c92e79b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.io.EnumSetWritable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
/**
* Race between two threads simultaneously calling
@@ -88,25 +89,40 @@ public class TestAddBlockRetry {
// start first addBlock()
LOG.info("Starting first addBlock for " + src);
LocatedBlock[] onRetryBlock = new LocatedBlock[1];
- DatanodeStorageInfo targets[] = ns.getNewBlockTargets(
- src, HdfsConstants.GRANDFATHER_INODE_ID, "clientName",
- null, null, null, onRetryBlock);
+ ns.readLock();
+ FSDirWriteFileOp.ValidateAddBlockResult r;
+ FSPermissionChecker pc = Mockito.mock(FSPermissionChecker.class);
+ try {
+ r = FSDirWriteFileOp.validateAddBlock(ns, pc, src,
+ HdfsConstants.GRANDFATHER_INODE_ID,
+ "clientName", null, onRetryBlock);
+ } finally {
+ ns.readUnlock();;
+ }
+ DatanodeStorageInfo targets[] = FSDirWriteFileOp.chooseTargetForNewBlock(
+ ns.getBlockManager(), src, null, null, r);
assertNotNull("Targets must be generated", targets);
// run second addBlock()
LOG.info("Starting second addBlock for " + src);
nn.addBlock(src, "clientName", null, null,
- HdfsConstants.GRANDFATHER_INODE_ID, null);
+ HdfsConstants.GRANDFATHER_INODE_ID, null);
assertTrue("Penultimate block must be complete",
- checkFileProgress(src, false));
+ checkFileProgress(src, false));
LocatedBlocks lbs = nn.getBlockLocations(src, 0, Long.MAX_VALUE);
assertEquals("Must be one block", 1, lbs.getLocatedBlocks().size());
LocatedBlock lb2 = lbs.get(0);
assertEquals("Wrong replication", REPLICATION, lb2.getLocations().length);
// continue first addBlock()
- LocatedBlock newBlock = ns.storeAllocatedBlock(
- src, HdfsConstants.GRANDFATHER_INODE_ID, "clientName", null, targets);
+ ns.writeLock();
+ LocatedBlock newBlock;
+ try {
+ newBlock = FSDirWriteFileOp.storeAllocatedBlock(ns, src,
+ HdfsConstants.GRANDFATHER_INODE_ID, "clientName", null, targets);
+ } finally {
+ ns.writeUnlock();
+ }
assertEquals("Blocks are not equal", lb2.getBlock(), newBlock.getBlock());
// check locations
http://git-wip-us.apache.org/repos/asf/hadoop/blob/480b65c7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
index 3049612..ea560fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderCon
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.junit.Test;
+import org.mockito.internal.util.reflection.Whitebox;
import java.io.IOException;
@@ -45,7 +46,9 @@ public class TestCommitBlockSynchronization {
private FSNamesystem makeNameSystemSpy(Block block, INodeFile file)
throws IOException {
Configuration conf = new Configuration();
+ FSEditLog editlog = mock(FSEditLog.class);
FSImage image = new FSImage(conf);
+ Whitebox.setInternalState(image, "editLog", editlog);
final DatanodeStorageInfo[] targets = {};
FSNamesystem namesystem = new FSNamesystem(conf, image);
[25/50] [abbrv] hadoop git commit: YARN-3601. Fix UT
TestRMFailover.testRMWebAppRedirect. Contributed by Weiwei Yang
Posted by zj...@apache.org.
YARN-3601. Fix UT TestRMFailover.testRMWebAppRedirect. Contributed by Weiwei Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b22cb5e2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b22cb5e2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b22cb5e2
Branch: refs/heads/YARN-2928
Commit: b22cb5e2f9588aee40ad2cddae78586af0e9a32d
Parents: 8812d83
Author: Xuan <xg...@apache.org>
Authored: Tue May 19 09:56:01 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:52:09 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 +
.../hadoop/yarn/client/TestRMFailover.java | 78 ++++++++++----------
2 files changed, 40 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b22cb5e2/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 17e0618..a2f3914 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -590,6 +590,8 @@ Release 2.7.1 - UNRELEASED
YARN-3526. ApplicationMaster tracking URL is incorrectly redirected
on a QJM cluster. (Weiwei Yang via xgong)
+ YARN-3601. Fix UT TestRMFailover.testRMWebAppRedirect. (Weiwei Yang via xgong)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b22cb5e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
index d4fc5c1..cd22743 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
@@ -21,13 +21,13 @@ package org.apache.hadoop.yarn.client;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
-import java.util.List;
-import java.util.Map;
+import javax.servlet.http.HttpServletResponse;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -48,7 +48,6 @@ import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
public class TestRMFailover extends ClientBaseWithFixes {
@@ -275,10 +274,6 @@ public class TestRMFailover extends ClientBaseWithFixes {
assertEquals(404, response.getResponseCode());
}
- // ignore this testcase, Always gets "too many redirect loops" exception
- // Probably because of the limitation of MiniYARNCluster.
- // Verified the behavior in a single node cluster.
- @Ignore
@Test
public void testRMWebAppRedirect() throws YarnException,
InterruptedException, IOException {
@@ -290,59 +285,62 @@ public class TestRMFailover extends ClientBaseWithFixes {
getAdminService(0).transitionToActive(req);
String rm1Url = "http://0.0.0.0:18088";
String rm2Url = "http://0.0.0.0:28088";
- String header = getHeader("Refresh", rm2Url);
- assertTrue(header.contains("; url=" + rm1Url));
+ String redirectURL = getRedirectURL(rm2Url);
+ // if uri is null, RMWebAppFilter will append a slash at the trail of the redirection url
+ assertEquals(redirectURL,rm1Url+"/");
- header = getHeader("Refresh", rm2Url + "/metrics");
- assertTrue(header.contains("; url=" + rm1Url));
+ redirectURL = getRedirectURL(rm2Url + "/metrics");
+ assertEquals(redirectURL,rm1Url + "/metrics");
- header = getHeader("Refresh", rm2Url + "/jmx");
- assertTrue(header.contains("; url=" + rm1Url));
+ redirectURL = getRedirectURL(rm2Url + "/jmx");
+ assertEquals(redirectURL,rm1Url + "/jmx");
// standby RM links /conf, /stacks, /logLevel, /static, /logs,
// /cluster/cluster as well as webService
// /ws/v1/cluster/info should not be redirected to active RM
- header = getHeader("Refresh", rm2Url + "/cluster/cluster");
- assertEquals(null, header);
+ redirectURL = getRedirectURL(rm2Url + "/cluster/cluster");
+ assertNull(redirectURL);
- header = getHeader("Refresh", rm2Url + "/conf");
- assertEquals(null, header);
+ redirectURL = getRedirectURL(rm2Url + "/conf");
+ assertNull(redirectURL);
- header = getHeader("Refresh", rm2Url + "/stacks");
- assertEquals(null, header);
+ redirectURL = getRedirectURL(rm2Url + "/stacks");
+ assertNull(redirectURL);
- header = getHeader("Refresh", rm2Url + "/logLevel");
- assertEquals(null, header);
+ redirectURL = getRedirectURL(rm2Url + "/logLevel");
+ assertNull(redirectURL);
- header = getHeader("Refresh", rm2Url + "/static");
- assertEquals(null, header);
+ redirectURL = getRedirectURL(rm2Url + "/static");
+ assertNull(redirectURL);
- header = getHeader("Refresh", rm2Url + "/logs");
- assertEquals(null, header);
+ redirectURL = getRedirectURL(rm2Url + "/logs");
+ assertNull(redirectURL);
- header = getHeader("Refresh", rm2Url + "/ws/v1/cluster/info");
- assertEquals(null, header);
+ redirectURL = getRedirectURL(rm2Url + "/ws/v1/cluster/info");
+ assertNull(redirectURL);
- header = getHeader("Refresh", rm2Url + "/ws/v1/cluster/apps");
- assertTrue(header.contains("; url=" + rm1Url));
+ redirectURL = getRedirectURL(rm2Url + "/ws/v1/cluster/apps");
+ assertEquals(redirectURL, rm1Url + "/ws/v1/cluster/apps");
- header = getHeader("Refresh", rm2Url + "/proxy/" + fakeAppId);
- assertEquals(null, header);
-
- // Due to the limitation of MiniYARNCluster and dispatcher is a singleton,
- // we couldn't add the test case after explicitFailover();
+ redirectURL = getRedirectURL(rm2Url + "/proxy/" + fakeAppId);
+ assertNull(redirectURL);
}
- static String getHeader(String field, String url) {
- String fieldHeader = null;
+ // set up http connection with the given url and get the redirection url from the response
+ // return null if the url is not redirected
+ static String getRedirectURL(String url) {
+ String redirectUrl = null;
try {
- Map<String, List<String>> map =
- new URL(url).openConnection().getHeaderFields();
- fieldHeader = map.get(field).get(0);
+ HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
+ // do not automatically follow the redirection
+ // otherwise we get too many redirections exception
+ conn.setInstanceFollowRedirects(false);
+ if(conn.getResponseCode() == HttpServletResponse.SC_TEMPORARY_REDIRECT)
+ redirectUrl = conn.getHeaderField("Location");
} catch (Exception e) {
// throw new RuntimeException(e);
}
- return fieldHeader;
+ return redirectUrl;
}
}
[07/50] [abbrv] hadoop git commit: HADOOP-11939. Deprecate DistCpV1
and Logalyzer. Contributed by Brahma Reddy Battula.
Posted by zj...@apache.org.
HADOOP-11939. Deprecate DistCpV1 and Logalyzer. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/24ac9db1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/24ac9db1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/24ac9db1
Branch: refs/heads/YARN-2928
Commit: 24ac9db109a2b267f7eaed945e73feeabab057a4
Parents: 6cb225b
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon May 18 16:13:16 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:47:01 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/tools/DistCpV1.java | 1 +
.../java/org/apache/hadoop/tools/Logalyzer.java | 3 ++-
.../org/apache/hadoop/tools/TestCopyFiles.java | 22 +++++++++++++++++---
.../org/apache/hadoop/tools/TestLogalyzer.java | 1 +
5 files changed, 26 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24ac9db1/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 5666035..7349091 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -572,6 +572,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11960. Enable Azure-Storage Client Side logging.
(Dushyanth via cnauroth)
+ HADOOP-11939. Deprecate DistCpV1 and Logalyzer.
+ (Brahma Reddy Battula via aajisaka)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24ac9db1/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCpV1.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCpV1.java b/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCpV1.java
index 8a6819b..39ac5c3 100644
--- a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCpV1.java
+++ b/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCpV1.java
@@ -81,6 +81,7 @@ import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
* A Map-reduce program to recursively copy directories between
* different file-systems.
*/
+@Deprecated
public class DistCpV1 implements Tool {
public static final Log LOG = LogFactory.getLog(DistCpV1.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24ac9db1/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java b/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java
index 449ecbf..05e6e24 100644
--- a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java
+++ b/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java
@@ -65,7 +65,7 @@ import org.apache.hadoop.mapreduce.lib.map.RegexMapper;
* <col1, col2> -separator <separator>
* <p>
*/
-
+@Deprecated
public class Logalyzer {
// Constants
private static Configuration fsConfig = new Configuration();
@@ -194,6 +194,7 @@ public class Logalyzer {
* @param archiveDirectory : The directory to store archived logfiles.
* @throws IOException
*/
+ @SuppressWarnings("deprecation")
public void
doArchive(String logListURI, String archiveDirectory)
throws IOException
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24ac9db1/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java b/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
index 73bba66..20b8ee2 100644
--- a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
+++ b/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
@@ -55,7 +55,7 @@ import org.apache.log4j.Level;
/**
* A JUnit test for copying files recursively.
*/
-
+@SuppressWarnings("deprecation")
public class TestCopyFiles extends TestCase {
{
((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.StateChange")
@@ -261,6 +261,7 @@ public class TestCopyFiles extends TestCase {
}
/** copy files from local file system to local file system */
+ @SuppressWarnings("deprecation")
public void testCopyFromLocalToLocal() throws Exception {
Configuration conf = new Configuration();
FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
@@ -275,6 +276,7 @@ public class TestCopyFiles extends TestCase {
}
/** copy files from dfs file system to dfs file system */
+ @SuppressWarnings("deprecation")
public void testCopyFromDfsToDfs() throws Exception {
String namenode = null;
MiniDFSCluster cluster = null;
@@ -305,6 +307,7 @@ public class TestCopyFiles extends TestCase {
}
/** copy empty directory on dfs file system */
+ @SuppressWarnings("deprecation")
public void testEmptyDir() throws Exception {
String namenode = null;
MiniDFSCluster cluster = null;
@@ -336,6 +339,7 @@ public class TestCopyFiles extends TestCase {
}
/** copy files from local file system to dfs file system */
+ @SuppressWarnings("deprecation")
public void testCopyFromLocalToDfs() throws Exception {
MiniDFSCluster cluster = null;
try {
@@ -364,6 +368,7 @@ public class TestCopyFiles extends TestCase {
}
/** copy files from dfs file system to local file system */
+ @SuppressWarnings("deprecation")
public void testCopyFromDfsToLocal() throws Exception {
MiniDFSCluster cluster = null;
try {
@@ -392,6 +397,7 @@ public class TestCopyFiles extends TestCase {
}
}
+ @SuppressWarnings("deprecation")
public void testCopyDfsToDfsUpdateOverwrite() throws Exception {
MiniDFSCluster cluster = null;
try {
@@ -452,6 +458,7 @@ public class TestCopyFiles extends TestCase {
}
}
+ @SuppressWarnings("deprecation")
public void testCopyDfsToDfsUpdateWithSkipCRC() throws Exception {
MiniDFSCluster cluster = null;
try {
@@ -526,7 +533,8 @@ public class TestCopyFiles extends TestCase {
if (cluster != null) { cluster.shutdown(); }
}
}
-
+
+ @SuppressWarnings("deprecation")
public void testCopyDuplication() throws Exception {
final FileSystem localfs = FileSystem.get(LOCAL_FS, new Configuration());
try {
@@ -550,6 +558,7 @@ public class TestCopyFiles extends TestCase {
}
}
+ @SuppressWarnings("deprecation")
public void testCopySingleFile() throws Exception {
FileSystem fs = FileSystem.get(LOCAL_FS, new Configuration());
Path root = new Path(TEST_ROOT_DIR+"/srcdat");
@@ -609,6 +618,7 @@ public class TestCopyFiles extends TestCase {
}
/** tests basedir option copying files from dfs file system to dfs file system */
+ @SuppressWarnings("deprecation")
public void testBasedir() throws Exception {
String namenode = null;
MiniDFSCluster cluster = null;
@@ -634,7 +644,8 @@ public class TestCopyFiles extends TestCase {
if (cluster != null) { cluster.shutdown(); }
}
}
-
+
+ @SuppressWarnings("deprecation")
public void testPreserveOption() throws Exception {
Configuration conf = new Configuration();
MiniDFSCluster cluster = null;
@@ -733,6 +744,7 @@ public class TestCopyFiles extends TestCase {
}
}
+ @SuppressWarnings("deprecation")
public void testMapCount() throws Exception {
String namenode = null;
MiniDFSCluster dfs = null;
@@ -787,6 +799,7 @@ public class TestCopyFiles extends TestCase {
}
}
+ @SuppressWarnings("deprecation")
public void testLimits() throws Exception {
Configuration conf = new Configuration();
MiniDFSCluster cluster = null;
@@ -892,6 +905,7 @@ public class TestCopyFiles extends TestCase {
}
/** test -delete */
+ @SuppressWarnings("deprecation")
public void testDelete() throws Exception {
final Configuration conf = new Configuration();
conf.setInt("fs.trash.interval", 60);
@@ -964,6 +978,7 @@ public class TestCopyFiles extends TestCase {
/**
* verify that -delete option works for other {@link FileSystem}
* implementations. See MAPREDUCE-1285 */
+ @SuppressWarnings("deprecation")
public void testDeleteLocal() throws Exception {
MiniDFSCluster cluster = null;
try {
@@ -997,6 +1012,7 @@ public class TestCopyFiles extends TestCase {
}
/** test globbing */
+ @SuppressWarnings("deprecation")
public void testGlobbing() throws Exception {
String namenode = null;
MiniDFSCluster cluster = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24ac9db1/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestLogalyzer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestLogalyzer.java b/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestLogalyzer.java
index 2042604..3787f6c 100644
--- a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestLogalyzer.java
+++ b/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestLogalyzer.java
@@ -44,6 +44,7 @@ public class TestLogalyzer {
+ File.separator + "out");
@Test
+ @SuppressWarnings("deprecation")
public void testLogalyzer() throws Exception {
Path f = createLogFile();
[30/50] [abbrv] hadoop git commit: HADOOP-11973. Ensure
ZkDelegationTokenSecretManager namespace znodes get created with ACLs.
(Gregory Chanan via asuresh)
Posted by zj...@apache.org.
HADOOP-11973. Ensure ZkDelegationTokenSecretManager namespace znodes get created with ACLs. (Gregory Chanan via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b31e61a6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b31e61a6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b31e61a6
Branch: refs/heads/YARN-2928
Commit: b31e61a6baed1c64f50f86781faa74bf7aaa0554
Parents: 08ddc0c
Author: Arun Suresh <as...@apache.org>
Authored: Tue May 19 11:35:57 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:52:10 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../ZKDelegationTokenSecretManager.java | 12 ++++
.../TestZKDelegationTokenSecretManager.java | 60 ++++++++++++++++++++
3 files changed, 75 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b31e61a6/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 10da9d7..e4537a3 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -748,6 +748,9 @@ Release 2.7.1 - UNRELEASED
HADOOP-11966. Variable cygwin is undefined in hadoop-config.sh when executed
through hadoop-daemon.sh. (cnauroth)
+ HADOOP-11973. Ensure ZkDelegationTokenSecretManager namespace znodes get
+ created with ACLs. (Gregory Chanan via asuresh)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b31e61a6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
index 73c3ab8..da0e6ad 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
@@ -47,6 +47,7 @@ import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
import org.apache.curator.framework.recipes.shared.SharedCount;
import org.apache.curator.framework.recipes.shared.VersionedValue;
import org.apache.curator.retry.RetryNTimes;
+import org.apache.curator.utils.EnsurePath;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -298,6 +299,17 @@ public abstract class ZKDelegationTokenSecretManager<TokenIdent extends Abstract
} catch (Exception e) {
throw new IOException("Could not start Curator Framework", e);
}
+ } else {
+ // If namespace parents are implicitly created, they won't have ACLs.
+ // So, let's explicitly create them.
+ CuratorFramework nullNsFw = zkClient.usingNamespace(null);
+ EnsurePath ensureNs =
+ nullNsFw.newNamespaceAwareEnsurePath("/" + zkClient.getNamespace());
+ try {
+ ensureNs.ensure(nullNsFw.getZookeeperClient());
+ } catch (Exception e) {
+ throw new IOException("Could not create namespace", e);
+ }
}
listenerThreadPool = Executors.newSingleThreadExecutor();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b31e61a6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java
index 6435c0b..185a994 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestZKDelegationTokenSecretManager.java
@@ -19,9 +19,16 @@
package org.apache.hadoop.security.token.delegation;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.framework.api.ACLProvider;
+import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.curator.test.TestingServer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
@@ -30,6 +37,10 @@ import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
import org.apache.hadoop.security.token.delegation.web.DelegationTokenManager;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Id;
+import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -296,6 +307,55 @@ public class TestZKDelegationTokenSecretManager {
tm1.destroy();
}
+ @Test
+ public void testACLs() throws Exception {
+ DelegationTokenManager tm1;
+ String connectString = zkServer.getConnectString();
+ Configuration conf = getSecretConf(connectString);
+ RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+ String userPass = "myuser:mypass";
+ final ACL digestACL = new ACL(ZooDefs.Perms.ALL, new Id("digest",
+ DigestAuthenticationProvider.generateDigest(userPass)));
+ ACLProvider digestAclProvider = new ACLProvider() {
+ @Override
+ public List<ACL> getAclForPath(String path) { return getDefaultAcl(); }
+
+ @Override
+ public List<ACL> getDefaultAcl() {
+ List<ACL> ret = new ArrayList<ACL>();
+ ret.add(digestACL);
+ return ret;
+ }
+ };
+
+ CuratorFramework curatorFramework =
+ CuratorFrameworkFactory.builder()
+ .connectString(connectString)
+ .retryPolicy(retryPolicy)
+ .aclProvider(digestAclProvider)
+ .authorization("digest", userPass.getBytes("UTF-8"))
+ .build();
+ curatorFramework.start();
+ ZKDelegationTokenSecretManager.setCurator(curatorFramework);
+ tm1 = new DelegationTokenManager(conf, new Text("bla"));
+ tm1.init();
+
+ // check ACL
+ String workingPath = conf.get(ZKDelegationTokenSecretManager.ZK_DTSM_ZNODE_WORKING_PATH);
+ verifyACL(curatorFramework, "/" + workingPath, digestACL);
+
+ tm1.destroy();
+ ZKDelegationTokenSecretManager.setCurator(null);
+ curatorFramework.close();
+ }
+
+ private void verifyACL(CuratorFramework curatorFramework,
+ String path, ACL expectedACL) throws Exception {
+ List<ACL> acls = curatorFramework.getACL().forPath(path);
+ Assert.assertEquals(1, acls.size());
+ Assert.assertEquals(expectedACL, acls.get(0));
+ }
+
// Since it is possible that there can be a delay for the cancel token message
// initiated by one node to reach another node.. The second node can ofcourse
// verify with ZK directly if the token that needs verification has been
[35/50] [abbrv] hadoop git commit: YARN-3565.
NodeHeartbeatRequest/RegisterNodeManagerRequest should use NodeLabel object
instead of String. (Naganarasimha G R via wangda)
Posted by zj...@apache.org.
YARN-3565. NodeHeartbeatRequest/RegisterNodeManagerRequest should use NodeLabel object instead of String. (Naganarasimha G R via wangda)
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69e6c7bc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69e6c7bc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69e6c7bc
Branch: refs/heads/YARN-2928
Commit: 69e6c7bc2afcb33fbc1661ee35c1f3f1a227e177
Parents: 48e840f
Author: Wangda Tan <wa...@apache.org>
Authored: Tue May 19 16:34:17 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 12:42:47 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../src/main/proto/yarn_protos.proto | 4 ---
.../nodelabels/CommonNodeLabelsManager.java | 2 ++
.../yarn/nodelabels/NodeLabelTestBase.java | 12 +++++++
.../protocolrecords/NodeHeartbeatRequest.java | 9 ++---
.../RegisterNodeManagerRequest.java | 7 ++--
.../impl/pb/NodeHeartbeatRequestPBImpl.java | 38 ++++++++++++++------
.../pb/RegisterNodeManagerRequestPBImpl.java | 35 +++++++++++++-----
.../yarn_server_common_service_protos.proto | 8 +++--
.../hadoop/yarn/TestYarnServerApiClasses.java | 19 +++++-----
.../nodemanager/NodeStatusUpdaterImpl.java | 23 ++++++------
.../nodelabels/NodeLabelsProvider.java | 3 +-
.../TestNodeStatusUpdaterForLabels.java | 23 ++++++------
.../resourcemanager/ResourceTrackerService.java | 18 ++++++++--
.../TestResourceTrackerService.java | 25 ++++++-------
15 files changed, 152 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69e6c7bc/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 7fbc09a..b6b6c9c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -328,6 +328,9 @@ Release 2.8.0 - UNRELEASED
YARN-3362. Add node label usage in RM CapacityScheduler web UI.
(Naganarasimha G R via wangda)
+ YARN-3565. NodeHeartbeatRequest/RegisterNodeManagerRequest should use
+ NodeLabel object instead of String. (Naganarasimha G R via wangda)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69e6c7bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 4095676..3c4aa52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -253,10 +253,6 @@ message NodeIdToLabelsProto {
repeated string nodeLabels = 2;
}
-message StringArrayProto {
- repeated string elements = 1;
-}
-
message LabelsToNodeIdsProto {
optional string nodeLabels = 1;
repeated NodeIdProto nodeId = 2;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69e6c7bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index bf34837..badf4d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -39,6 +39,7 @@ import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.NodeId;
@@ -59,6 +60,7 @@ import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
+@Private
public class CommonNodeLabelsManager extends AbstractService {
protected static final Log LOG = LogFactory.getLog(CommonNodeLabelsManager.class);
private static final int MAX_LABEL_LENGTH = 255;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69e6c7bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java
index 8301d96..f834d54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java
@@ -112,6 +112,18 @@ public class NodeLabelTestBase {
return set;
}
+ @SuppressWarnings("unchecked")
+ public static Set<NodeLabel> toNodeLabelSet(String... nodeLabelsStr) {
+ if (null == nodeLabelsStr) {
+ return null;
+ }
+ Set<NodeLabel> labels = new HashSet<NodeLabel>();
+ for (String label : nodeLabelsStr) {
+ labels.add(NodeLabel.newInstance(label));
+ }
+ return labels;
+ }
+
public NodeId toNodeId(String str) {
if (str.contains(":")) {
int idx = str.indexOf(':');
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69e6c7bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
index ce42d2e..7df9cf4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
@@ -23,6 +23,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import org.apache.hadoop.yarn.util.Records;
@@ -31,7 +32,7 @@ public abstract class NodeHeartbeatRequest {
public static NodeHeartbeatRequest newInstance(NodeStatus nodeStatus,
MasterKey lastKnownContainerTokenMasterKey,
- MasterKey lastKnownNMTokenMasterKey, Set<String> nodeLabels) {
+ MasterKey lastKnownNMTokenMasterKey, Set<NodeLabel> nodeLabels) {
NodeHeartbeatRequest nodeHeartbeatRequest =
Records.newRecord(NodeHeartbeatRequest.class);
nodeHeartbeatRequest.setNodeStatus(nodeStatus);
@@ -45,7 +46,7 @@ public abstract class NodeHeartbeatRequest {
public static NodeHeartbeatRequest newInstance(NodeStatus nodeStatus,
MasterKey lastKnownContainerTokenMasterKey,
- MasterKey lastKnownNMTokenMasterKey, Set<String> nodeLabels,
+ MasterKey lastKnownNMTokenMasterKey, Set<NodeLabel> nodeLabels,
Map<ApplicationId, String> registeredCollectors) {
NodeHeartbeatRequest nodeHeartbeatRequest =
NodeHeartbeatRequest.newInstance(nodeStatus,
@@ -65,8 +66,8 @@ public abstract class NodeHeartbeatRequest {
public abstract MasterKey getLastKnownNMTokenMasterKey();
public abstract void setLastKnownNMTokenMasterKey(MasterKey secretKey);
- public abstract Set<String> getNodeLabels();
- public abstract void setNodeLabels(Set<String> nodeLabels);
+ public abstract Set<NodeLabel> getNodeLabels();
+ public abstract void setNodeLabels(Set<NodeLabel> nodeLabels);
public abstract List<LogAggregationReport>
getLogAggregationReportsForApps();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69e6c7bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
index bf09b33..7798ba9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
@@ -23,6 +23,7 @@ import java.util.Set;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.Records;
@@ -39,7 +40,7 @@ public abstract class RegisterNodeManagerRequest {
public static RegisterNodeManagerRequest newInstance(NodeId nodeId,
int httpPort, Resource resource, String nodeManagerVersionId,
List<NMContainerStatus> containerStatuses,
- List<ApplicationId> runningApplications, Set<String> nodeLabels) {
+ List<ApplicationId> runningApplications, Set<NodeLabel> nodeLabels) {
RegisterNodeManagerRequest request =
Records.newRecord(RegisterNodeManagerRequest.class);
request.setHttpPort(httpPort);
@@ -57,8 +58,8 @@ public abstract class RegisterNodeManagerRequest {
public abstract Resource getResource();
public abstract String getNMVersion();
public abstract List<NMContainerStatus> getNMContainerStatuses();
- public abstract Set<String> getNodeLabels();
- public abstract void setNodeLabels(Set<String> nodeLabels);
+ public abstract Set<NodeLabel> getNodeLabels();
+ public abstract void setNodeLabels(Set<NodeLabel> nodeLabels);
/**
* We introduce this here because currently YARN RM doesn't persist nodes info
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69e6c7bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
index 81dcdc2..56908e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
@@ -28,16 +28,19 @@ import java.util.Map.Entry;
import java.util.Set;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdToLabelsProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.StringArrayProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.LogAggregationReportProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeLabelsProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeLabelsProto.Builder;
import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
@@ -53,7 +56,7 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
private NodeStatus nodeStatus = null;
private MasterKey lastKnownContainerTokenMasterKey = null;
private MasterKey lastKnownNMTokenMasterKey = null;
- private Set<String> labels = null;
+ private Set<NodeLabel> labels = null;
private List<LogAggregationReport> logAggregationReportsForApps = null;
Map<ApplicationId, String> registeredCollectors = null;
@@ -102,8 +105,11 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
}
if (this.labels != null) {
builder.clearNodeLabels();
- builder.setNodeLabels(StringArrayProto.newBuilder()
- .addAllElements(this.labels).build());
+ Builder newBuilder = NodeLabelsProto.newBuilder();
+ for (NodeLabel label : labels) {
+ newBuilder.addNodeLabels(convertToProtoFormat(label));
+ }
+ builder.setNodeLabels(newBuilder.build());
}
if (this.logAggregationReportsForApps != null) {
addLogAggregationStatusForAppsToProto();
@@ -150,7 +156,8 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
private void addRegisteredCollectorsToProto() {
maybeInitBuilder();
builder.clearRegisteredCollectors();
- for (Map.Entry<ApplicationId, String> entry : registeredCollectors.entrySet()) {
+ for (Map.Entry<ApplicationId, String> entry : registeredCollectors
+ .entrySet()) {
builder.addRegisteredCollectors(AppCollectorsMapProto.newBuilder()
.setAppId(convertToProtoFormat(entry.getKey()))
.setAppCollectorAddr(entry.getValue()));
@@ -290,13 +297,13 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
}
@Override
- public Set<String> getNodeLabels() {
+ public Set<NodeLabel> getNodeLabels() {
initNodeLabels();
return this.labels;
}
@Override
- public void setNodeLabels(Set<String> nodeLabels) {
+ public void setNodeLabels(Set<NodeLabel> nodeLabels) {
maybeInitBuilder();
builder.clearNodeLabels();
this.labels = nodeLabels;
@@ -311,8 +318,11 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
labels = null;
return;
}
- StringArrayProto nodeLabels = p.getNodeLabels();
- labels = new HashSet<String>(nodeLabels.getElementsList());
+ NodeLabelsProto nodeLabels = p.getNodeLabels();
+ labels = new HashSet<NodeLabel>();
+ for(NodeLabelProto nlp : nodeLabels.getNodeLabelsList()) {
+ labels.add(convertFromProtoFormat(nlp));
+ }
}
private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) {
@@ -323,6 +333,14 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
return ((ApplicationIdPBImpl) t).getProto();
}
+ private NodeLabelPBImpl convertFromProtoFormat(NodeLabelProto p) {
+ return new NodeLabelPBImpl(p);
+ }
+
+ private NodeLabelProto convertToProtoFormat(NodeLabel t) {
+ return ((NodeLabelPBImpl)t).getProto();
+ }
+
@Override
public List<LogAggregationReport> getLogAggregationReportsForApps() {
if (this.logAggregationReportsForApps != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69e6c7bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
index 1d2bb82..5b0e0a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
@@ -27,16 +27,19 @@ import java.util.Set;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdToLabelsProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.StringArrayProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NMContainerStatusProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeLabelsProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeLabelsProto.Builder;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.RegisterNodeManagerRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.RegisterNodeManagerRequestProtoOrBuilder;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
@@ -51,7 +54,7 @@ public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest
private NodeId nodeId = null;
private List<NMContainerStatus> containerStatuses = null;
private List<ApplicationId> runningApplications = null;
- private Set<String> labels = null;
+ private Set<NodeLabel> labels = null;
public RegisterNodeManagerRequestPBImpl() {
builder = RegisterNodeManagerRequestProto.newBuilder();
@@ -84,8 +87,11 @@ public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest
}
if (this.labels != null) {
builder.clearNodeLabels();
- builder.setNodeLabels(StringArrayProto.newBuilder()
- .addAllElements(this.labels).build());
+ Builder newBuilder = NodeLabelsProto.newBuilder();
+ for (NodeLabel label : labels) {
+ newBuilder.addNodeLabels(convertToProtoFormat(label));
+ }
+ builder.setNodeLabels(newBuilder.build());
}
}
@@ -293,13 +299,13 @@ public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest
}
@Override
- public Set<String> getNodeLabels() {
+ public Set<NodeLabel> getNodeLabels() {
initNodeLabels();
return this.labels;
}
@Override
- public void setNodeLabels(Set<String> nodeLabels) {
+ public void setNodeLabels(Set<NodeLabel> nodeLabels) {
maybeInitBuilder();
builder.clearNodeLabels();
this.labels = nodeLabels;
@@ -314,8 +320,19 @@ public class RegisterNodeManagerRequestPBImpl extends RegisterNodeManagerRequest
labels=null;
return;
}
- StringArrayProto nodeLabels = p.getNodeLabels();
- labels = new HashSet<String>(nodeLabels.getElementsList());
+ NodeLabelsProto nodeLabels = p.getNodeLabels();
+ labels = new HashSet<NodeLabel>();
+ for(NodeLabelProto nlp : nodeLabels.getNodeLabelsList()) {
+ labels.add(convertFromProtoFormat(nlp));
+ }
+ }
+
+ private NodeLabelPBImpl convertFromProtoFormat(NodeLabelProto p) {
+ return new NodeLabelPBImpl(p);
+ }
+
+ private NodeLabelProto convertToProtoFormat(NodeLabel t) {
+ return ((NodeLabelPBImpl)t).getProto();
}
private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69e6c7bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
index 50cb430..94dd46a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
@@ -25,6 +25,10 @@ package hadoop.yarn;
import "yarn_protos.proto";
import "yarn_server_common_protos.proto";
+message NodeLabelsProto {
+ repeated NodeLabelProto nodeLabels = 1;
+}
+
message RegisterNodeManagerRequestProto {
optional NodeIdProto node_id = 1;
optional int32 http_port = 3;
@@ -32,7 +36,7 @@ message RegisterNodeManagerRequestProto {
optional string nm_version = 5;
repeated NMContainerStatusProto container_statuses = 6;
repeated ApplicationIdProto runningApplications = 7;
- optional StringArrayProto nodeLabels = 8;
+ optional NodeLabelsProto nodeLabels = 8;
}
message RegisterNodeManagerResponseProto {
@@ -49,7 +53,7 @@ message NodeHeartbeatRequestProto {
optional NodeStatusProto node_status = 1;
optional MasterKeyProto last_known_container_token_master_key = 2;
optional MasterKeyProto last_known_nm_token_master_key = 3;
- optional StringArrayProto nodeLabels = 4;
+ optional NodeLabelsProto nodeLabels = 4;
repeated LogAggregationReportProto log_aggregation_reports_for_apps = 5;
repeated AppCollectorsMapProto registered_collectors = 6;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69e6c7bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
index 7d90c86..779bfee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
@@ -117,7 +118,7 @@ public class TestYarnServerApiClasses {
Assert.assertTrue(original.getNodeLabels()
.containsAll(copy.getNodeLabels()));
// check for empty labels
- original.setNodeLabels(new HashSet<String> ());
+ original.setNodeLabels(new HashSet<NodeLabel> ());
copy = new NodeHeartbeatRequestPBImpl(
original.getProto());
Assert.assertNotNull(copy.getNodeLabels());
@@ -288,7 +289,7 @@ public class TestYarnServerApiClasses {
@Test
public void testRegisterNodeManagerRequestWithValidLabels() {
- HashSet<String> nodeLabels = getValidNodeLabels();
+ HashSet<NodeLabel> nodeLabels = getValidNodeLabels();
RegisterNodeManagerRequest request =
RegisterNodeManagerRequest.newInstance(
NodeId.newInstance("host", 1234), 1234, Resource.newInstance(0, 0),
@@ -303,19 +304,19 @@ public class TestYarnServerApiClasses {
Assert.assertEquals(true, nodeLabels.containsAll(copy.getNodeLabels()));
// check for empty labels
- request.setNodeLabels(new HashSet<String> ());
+ request.setNodeLabels(new HashSet<NodeLabel> ());
copy = new RegisterNodeManagerRequestPBImpl(
((RegisterNodeManagerRequestPBImpl) request).getProto());
Assert.assertNotNull(copy.getNodeLabels());
Assert.assertEquals(0, copy.getNodeLabels().size());
}
- private HashSet<String> getValidNodeLabels() {
- HashSet<String> nodeLabels = new HashSet<String>();
- nodeLabels.add("java");
- nodeLabels.add("windows");
- nodeLabels.add("gpu");
- nodeLabels.add("x86");
+ private HashSet<NodeLabel> getValidNodeLabels() {
+ HashSet<NodeLabel> nodeLabels = new HashSet<NodeLabel>();
+ nodeLabels.add(NodeLabel.newInstance("java"));
+ nodeLabels.add(NodeLabel.newInstance("windows"));
+ nodeLabels.add(NodeLabel.newInstance("gpu"));
+ nodeLabels.add(NodeLabel.newInstance("x86"));
return nodeLabels;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69e6c7bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index 4b93111..86293cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -30,9 +30,9 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.ConcurrentLinkedQueue;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.client.api.TimelineClient;
@@ -281,11 +282,11 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
protected void registerWithRM()
throws YarnException, IOException {
List<NMContainerStatus> containerReports = getNMContainerStatuses();
- Set<String> nodeLabels = null;
+ Set<NodeLabel> nodeLabels = null;
if (hasNodeLabelsProvider) {
nodeLabels = nodeLabelsProvider.getNodeLabels();
nodeLabels =
- (null == nodeLabels) ? CommonNodeLabelsManager.EMPTY_STRING_SET
+ (null == nodeLabels) ? CommonNodeLabelsManager.EMPTY_NODELABEL_SET
: nodeLabels;
}
RegisterNodeManagerRequest request =
@@ -630,29 +631,29 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
@SuppressWarnings("unchecked")
public void run() {
int lastHeartbeatID = 0;
- Set<String> lastUpdatedNodeLabelsToRM = null;
+ Set<NodeLabel> lastUpdatedNodeLabelsToRM = null;
if (hasNodeLabelsProvider) {
lastUpdatedNodeLabelsToRM = nodeLabelsProvider.getNodeLabels();
lastUpdatedNodeLabelsToRM =
- (null == lastUpdatedNodeLabelsToRM) ? CommonNodeLabelsManager.EMPTY_STRING_SET
+ (null == lastUpdatedNodeLabelsToRM) ? CommonNodeLabelsManager.EMPTY_NODELABEL_SET
: lastUpdatedNodeLabelsToRM;
}
while (!isStopped) {
// Send heartbeat
try {
NodeHeartbeatResponse response = null;
- Set<String> nodeLabelsForHeartbeat = null;
+ Set<NodeLabel> nodeLabelsForHeartbeat = null;
NodeStatus nodeStatus = getNodeStatus(lastHeartbeatID);
if (hasNodeLabelsProvider) {
nodeLabelsForHeartbeat = nodeLabelsProvider.getNodeLabels();
- //if the provider returns null then consider empty labels are set
+ // if the provider returns null then consider empty labels are set
nodeLabelsForHeartbeat =
- (nodeLabelsForHeartbeat == null) ? CommonNodeLabelsManager.EMPTY_STRING_SET
+ (nodeLabelsForHeartbeat == null) ? CommonNodeLabelsManager.EMPTY_NODELABEL_SET
: nodeLabelsForHeartbeat;
if (!areNodeLabelsUpdated(nodeLabelsForHeartbeat,
lastUpdatedNodeLabelsToRM)) {
- //if nodelabels have not changed then no need to send
+ // if nodelabels have not changed then no need to send
nodeLabelsForHeartbeat = null;
}
}
@@ -811,8 +812,8 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
* @param nodeLabelsOld
* @return if the New node labels are diff from the older one.
*/
- private boolean areNodeLabelsUpdated(Set<String> nodeLabelsNew,
- Set<String> nodeLabelsOld) {
+ private boolean areNodeLabelsUpdated(Set<NodeLabel> nodeLabelsNew,
+ Set<NodeLabel> nodeLabelsOld) {
if (nodeLabelsNew.size() != nodeLabelsOld.size()
|| !nodeLabelsOld.containsAll(nodeLabelsNew)) {
return true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69e6c7bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeLabelsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeLabelsProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeLabelsProvider.java
index 4b34d76..dab3709 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeLabelsProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeLabelsProvider.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.nodemanager.nodelabels;
import java.util.Set;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
/**
* Interface which will be responsible for fetching the labels
@@ -39,5 +40,5 @@ public abstract class NodeLabelsProvider extends AbstractService {
*
* @return Set of node label strings applicable for a node
*/
- public abstract Set<String> getNodeLabels();
+ public abstract Set<NodeLabel> getNodeLabels();
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69e6c7bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdaterForLabels.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdaterForLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdaterForLabels.java
index 437e4c8..a0ed39b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdaterForLabels.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdaterForLabels.java
@@ -23,16 +23,17 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.Collections;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.ServiceOperations;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase;
import org.apache.hadoop.yarn.server.api.ResourceTracker;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
@@ -71,7 +72,7 @@ public class TestNodeStatusUpdaterForLabels extends NodeLabelTestBase {
private class ResourceTrackerForLabels implements ResourceTracker {
int heartbeatID = 0;
- Set<String> labels;
+ Set<NodeLabel> labels;
private boolean receivedNMHeartbeat = false;
private boolean receivedNMRegister = false;
@@ -185,18 +186,18 @@ public class TestNodeStatusUpdaterForLabels extends NodeLabelTestBase {
public static class DummyNodeLabelsProvider extends NodeLabelsProvider {
@SuppressWarnings("unchecked")
- private Set<String> nodeLabels = Collections.EMPTY_SET;
+ private Set<NodeLabel> nodeLabels = CommonNodeLabelsManager.EMPTY_NODELABEL_SET;
public DummyNodeLabelsProvider() {
super(DummyNodeLabelsProvider.class.getName());
}
@Override
- public synchronized Set<String> getNodeLabels() {
+ public synchronized Set<NodeLabel> getNodeLabels() {
return nodeLabels;
}
- synchronized void setNodeLabels(Set<String> nodeLabels) {
+ synchronized void setNodeLabels(Set<NodeLabel> nodeLabels) {
this.nodeLabels = nodeLabels;
}
}
@@ -245,19 +246,21 @@ public class TestNodeStatusUpdaterForLabels extends NodeLabelTestBase {
resourceTracker.resetNMHeartbeatReceiveFlag();
nm.start();
resourceTracker.waitTillRegister();
- assertCollectionEquals(resourceTracker.labels,
- dummyLabelsProviderRef.getNodeLabels());
+ assertNLCollectionEquals(resourceTracker.labels,
+ dummyLabelsProviderRef
+ .getNodeLabels());
resourceTracker.waitTillHeartbeat();// wait till the first heartbeat
resourceTracker.resetNMHeartbeatReceiveFlag();
// heartbeat with updated labels
- dummyLabelsProviderRef.setNodeLabels(toSet("P"));
+ dummyLabelsProviderRef.setNodeLabels(toNodeLabelSet("P"));
nm.getNodeStatusUpdater().sendOutofBandHeartBeat();
resourceTracker.waitTillHeartbeat();
- assertCollectionEquals(resourceTracker.labels,
- dummyLabelsProviderRef.getNodeLabels());
+ assertNLCollectionEquals(resourceTracker.labels,
+ dummyLabelsProviderRef
+ .getNodeLabels());
resourceTracker.resetNMHeartbeatReceiveFlag();
// heartbeat without updating labels
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69e6c7bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index 8018ee0..acb690b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -22,6 +22,7 @@ import java.io.InputStream;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -44,6 +45,7 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -250,6 +252,17 @@ public class ResourceTrackerService extends AbstractService implements
}
}
+ static Set<String> convertToStringSet(Set<NodeLabel> nodeLabels) {
+ if (null == nodeLabels) {
+ return null;
+ }
+ Set<String> labels = new HashSet<String>();
+ for (NodeLabel label : nodeLabels) {
+ labels.add(label.getName());
+ }
+ return labels;
+ }
+
@SuppressWarnings("unchecked")
@Override
public RegisterNodeManagerResponse registerNodeManager(
@@ -349,7 +362,7 @@ public class ResourceTrackerService extends AbstractService implements
}
// Update node's labels to RM's NodeLabelManager.
- Set<String> nodeLabels = request.getNodeLabels();
+ Set<String> nodeLabels = convertToStringSet(request.getNodeLabels());
if (isDistributedNodeLabelsConf && nodeLabels != null) {
try {
updateNodeLabelsFromNMReport(nodeLabels, nodeId);
@@ -484,7 +497,8 @@ public class ResourceTrackerService extends AbstractService implements
// 5. Update node's labels to RM's NodeLabelManager.
if (isDistributedNodeLabelsConf && request.getNodeLabels() != null) {
try {
- updateNodeLabelsFromNMReport(request.getNodeLabels(), nodeId);
+ updateNodeLabelsFromNMReport(
+ convertToStringSet(request.getNodeLabels()), nodeId);
nodeHeartBeatResponse.setAreNodeLabelsAcceptedByRM(true);
} catch (IOException ex) {
//ensure the error message is captured and sent across in response
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69e6c7bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index cc5f464..3474ed6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -353,14 +354,14 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
registerReq.setNodeId(nodeId);
registerReq.setHttpPort(1234);
registerReq.setNMVersion(YarnVersionInfo.getVersion());
- registerReq.setNodeLabels(toSet("A"));
+ registerReq.setNodeLabels(toSet(NodeLabel.newInstance("A")));
RegisterNodeManagerResponse response =
resourceTrackerService.registerNodeManager(registerReq);
Assert.assertEquals("Action should be normal on valid Node Labels",
NodeAction.NORMAL, response.getNodeAction());
assertCollectionEquals(nodeLabelsMgr.getNodeLabels().get(nodeId),
- registerReq.getNodeLabels());
+ ResourceTrackerService.convertToStringSet(registerReq.getNodeLabels()));
Assert.assertTrue("Valid Node Labels were not accepted by RM",
response.getAreNodeLabelsAcceptedByRM());
rm.stop();
@@ -402,7 +403,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
registerReq.setNodeId(nodeId);
registerReq.setHttpPort(1234);
registerReq.setNMVersion(YarnVersionInfo.getVersion());
- registerReq.setNodeLabels(toSet("A", "B", "C"));
+ registerReq.setNodeLabels(toNodeLabelSet("A", "B", "C"));
RegisterNodeManagerResponse response =
resourceTrackerService.registerNodeManager(registerReq);
@@ -455,7 +456,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
req.setNodeId(nodeId);
req.setHttpPort(1234);
req.setNMVersion(YarnVersionInfo.getVersion());
- req.setNodeLabels(toSet("#Y"));
+ req.setNodeLabels(toNodeLabelSet("#Y"));
RegisterNodeManagerResponse response =
resourceTrackerService.registerNodeManager(req);
@@ -506,7 +507,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
req.setNodeId(nodeId);
req.setHttpPort(1234);
req.setNMVersion(YarnVersionInfo.getVersion());
- req.setNodeLabels(toSet("A"));
+ req.setNodeLabels(toNodeLabelSet("A"));
RegisterNodeManagerResponse response =
resourceTrackerService.registerNodeManager(req);
// registered to RM with central label config
@@ -568,14 +569,14 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
registerReq.setNodeId(nodeId);
registerReq.setHttpPort(1234);
registerReq.setNMVersion(YarnVersionInfo.getVersion());
- registerReq.setNodeLabels(toSet("A")); // Node register label
+ registerReq.setNodeLabels(toNodeLabelSet("A")); // Node register label
RegisterNodeManagerResponse registerResponse =
resourceTrackerService.registerNodeManager(registerReq);
// modification of labels during heartbeat
NodeHeartbeatRequest heartbeatReq =
Records.newRecord(NodeHeartbeatRequest.class);
- heartbeatReq.setNodeLabels(toSet("B")); // Node heartbeat label update
+ heartbeatReq.setNodeLabels(toNodeLabelSet("B")); // Node heartbeat label update
NodeStatus nodeStatusObject = getNodeStatusObject(nodeId);
heartbeatReq.setNodeStatus(nodeStatusObject);
heartbeatReq.setLastKnownNMTokenMasterKey(registerResponse
@@ -588,7 +589,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
Assert.assertEquals("InValid Node Labels were not accepted by RM",
NodeAction.NORMAL, nodeHeartbeatResponse.getNodeAction());
assertCollectionEquals(nodeLabelsMgr.getNodeLabels().get(nodeId),
- heartbeatReq.getNodeLabels());
+ ResourceTrackerService.convertToStringSet(heartbeatReq.getNodeLabels()));
Assert.assertTrue("Valid Node Labels were not accepted by RM",
nodeHeartbeatResponse.getAreNodeLabelsAcceptedByRM());
@@ -652,13 +653,13 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
registerReq.setNodeId(nodeId);
registerReq.setHttpPort(1234);
registerReq.setNMVersion(YarnVersionInfo.getVersion());
- registerReq.setNodeLabels(toSet("A"));
+ registerReq.setNodeLabels(toNodeLabelSet("A"));
RegisterNodeManagerResponse registerResponse =
resourceTrackerService.registerNodeManager(registerReq);
NodeHeartbeatRequest heartbeatReq =
Records.newRecord(NodeHeartbeatRequest.class);
- heartbeatReq.setNodeLabels(toSet("B", "#C")); // Invalid heart beat labels
+ heartbeatReq.setNodeLabels(toNodeLabelSet("B", "#C")); // Invalid heart beat labels
heartbeatReq.setNodeStatus(getNodeStatusObject(nodeId));
heartbeatReq.setLastKnownNMTokenMasterKey(registerResponse
.getNMTokenMasterKey());
@@ -705,13 +706,13 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
req.setNodeId(nodeId);
req.setHttpPort(1234);
req.setNMVersion(YarnVersionInfo.getVersion());
- req.setNodeLabels(toSet("A", "B", "C"));
+ req.setNodeLabels(toNodeLabelSet("A", "B", "C"));
RegisterNodeManagerResponse registerResponse =
resourceTrackerService.registerNodeManager(req);
NodeHeartbeatRequest heartbeatReq =
Records.newRecord(NodeHeartbeatRequest.class);
- heartbeatReq.setNodeLabels(toSet("B")); // Valid heart beat labels
+ heartbeatReq.setNodeLabels(toNodeLabelSet("B")); // Valid heart beat labels
heartbeatReq.setNodeStatus(getNodeStatusObject(nodeId));
heartbeatReq.setLastKnownNMTokenMasterKey(registerResponse
.getNMTokenMasterKey());
[08/50] [abbrv] hadoop git commit: HADOOP-11988. Fix typo in the
document for hadoop fs -find. Contributed by Kengo Seki.
Posted by zj...@apache.org.
HADOOP-11988. Fix typo in the document for hadoop fs -find. Contributed by Kengo Seki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6cb225b7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6cb225b7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6cb225b7
Branch: refs/heads/YARN-2928
Commit: 6cb225b722bb3c7de9e42abf428b55e79ee15464
Parents: d01afb2
Author: Akira Ajisaka <aa...@apache.org>
Authored: Sun May 17 13:35:23 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:47:01 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop-common/src/site/markdown/FileSystemShell.md | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cb225b7/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index aecfde4..5666035 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -689,6 +689,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-8174. Remove confusing comment in Path#isAbsolute()
(Suresh Srinivas via vinayakumarb)
+ HADOOP-11988. Fix typo in the document for hadoop fs -find.
+ (Kengo Seki via aajisaka)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cb225b7/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 305370d..2920e01 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -286,9 +286,9 @@ The following primary expressions are recognised:
Evaluates as true if the basename of the file matches the pattern using standard file system globbing. If -iname is used then the match is case insensitive.
-* -print<br />-print0Always
+* -print<br />-print0
- evaluates to true. Causes the current pathname to be written to standard output. If the -print0 expression is used then an ASCII NULL character is appended.
+ Always evaluates to true. Causes the current pathname to be written to standard output. If the -print0 expression is used then an ASCII NULL character is appended.
The following operators are recognised:
[29/50] [abbrv] hadoop git commit: Moving MAPREDUCE-6361 to 2.7.1
CHANGES.txt
Posted by zj...@apache.org.
Moving MAPREDUCE-6361 to 2.7.1 CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/08ddc0c9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/08ddc0c9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/08ddc0c9
Branch: refs/heads/YARN-2928
Commit: 08ddc0c98c8bece6492eaeddf00e4f3e857b5aea
Parents: 6f2ba72
Author: Junping Du <ju...@apache.org>
Authored: Tue May 19 11:48:06 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:52:10 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08ddc0c9/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 40b829d..cdb10f1 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -439,10 +439,6 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6360. TestMapreduceConfigFields is placed in wrong dir,
introducing compile error (Arshad Mohammad via vinayakumarb)
- MAPREDUCE-6361. NPE issue in shuffle caused by concurrent issue between
- copySucceeded() in one thread and copyFailed() in another thread on the
- same host. (Junping Du via ozawa)
-
MAPREDUCE-6366. mapreduce.terasort.final.sync configuration in TeraSort
doesn't work. (Takuya Fukudome via ozawa)
@@ -489,6 +485,10 @@ Release 2.7.1 - UNRELEASED
that they don't fail on history-server backed by DFSes with not so strong
guarantees. (Craig Welch via vinodkv)
+ MAPREDUCE-6361. NPE issue in shuffle caused by concurrent issue between
+ copySucceeded() in one thread and copyFailed() in another thread on the
+ same host. (Junping Du via ozawa)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
[18/50] [abbrv] hadoop git commit: YARN-3541. Add version info on
timeline service / generic history web UI and REST API. Contributed by Zhijie
Shen
Posted by zj...@apache.org.
YARN-3541. Add version info on timeline service / generic history web UI and REST API. Contributed by Zhijie Shen
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f5fce6d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f5fce6d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f5fce6d
Branch: refs/heads/YARN-2928
Commit: 1f5fce6d191c3e574b3befb688fca83a40005fba
Parents: 7ef6b58
Author: Xuan <xg...@apache.org>
Authored: Mon May 18 13:17:16 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:51:59 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../api/records/timeline/TimelineAbout.java | 116 +++++++++++++++++++
.../yarn/util/timeline/TimelineUtils.java | 14 +++
.../webapp/AHSController.java | 4 +
.../webapp/AHSWebApp.java | 1 +
.../webapp/AHSWebServices.java | 12 ++
.../webapp/AboutBlock.java | 47 ++++++++
.../webapp/AboutPage.java | 36 ++++++
.../webapp/NavBlock.java | 2 +
.../timeline/webapp/TimelineWebServices.java | 41 +------
.../webapp/TestAHSWebApp.java | 14 +++
.../webapp/TestAHSWebServices.java | 31 +++++
.../webapp/TestTimelineWebServices.java | 25 +++-
.../src/site/markdown/TimelineServer.md | 101 +++++++++++++++-
14 files changed, 404 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f5fce6d/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 074aef0..17e0618 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -195,6 +195,9 @@ Release 2.8.0 - UNRELEASED
YARN-3505. Node's Log Aggregation Report with SUCCEED should not cached in
RMApps. (Xuan Gong via junping_du)
+ YARN-3541. Add version info on timeline service / generic history web UI
+ and REST API. (Zhijie Shen via xgong)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f5fce6d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineAbout.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineAbout.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineAbout.java
new file mode 100644
index 0000000..0a2625c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineAbout.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.timeline;
+
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement(name = "about")
+@XmlAccessorType(XmlAccessType.NONE)
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class TimelineAbout {
+
+ private String about;
+ private String timelineServiceVersion;
+ private String timelineServiceBuildVersion;
+ private String timelineServiceVersionBuiltOn;
+ private String hadoopVersion;
+ private String hadoopBuildVersion;
+ private String hadoopVersionBuiltOn;
+
+ public TimelineAbout() {
+ }
+
+ public TimelineAbout(String about) {
+ this.about = about;
+ }
+
+ @XmlElement(name = "About")
+ public String getAbout() {
+ return about;
+ }
+
+ public void setAbout(String about) {
+ this.about = about;
+ }
+
+ @XmlElement(name = "timeline-service-version")
+ public String getTimelineServiceVersion() {
+ return timelineServiceVersion;
+ }
+
+ public void setTimelineServiceVersion(String timelineServiceVersion) {
+ this.timelineServiceVersion = timelineServiceVersion;
+ }
+
+ @XmlElement(name = "timeline-service-build-version")
+ public String getTimelineServiceBuildVersion() {
+ return timelineServiceBuildVersion;
+ }
+
+ public void setTimelineServiceBuildVersion(
+ String timelineServiceBuildVersion) {
+ this.timelineServiceBuildVersion = timelineServiceBuildVersion;
+ }
+
+ @XmlElement(name = "timeline-service-version-built-on")
+ public String getTimelineServiceVersionBuiltOn() {
+ return timelineServiceVersionBuiltOn;
+ }
+
+ public void setTimelineServiceVersionBuiltOn(
+ String timelineServiceVersionBuiltOn) {
+ this.timelineServiceVersionBuiltOn = timelineServiceVersionBuiltOn;
+ }
+
+ @XmlElement(name = "hadoop-version")
+ public String getHadoopVersion() {
+ return hadoopVersion;
+ }
+
+ public void setHadoopVersion(String hadoopVersion) {
+ this.hadoopVersion = hadoopVersion;
+ }
+
+ @XmlElement(name = "hadoop-build-version")
+ public String getHadoopBuildVersion() {
+ return hadoopBuildVersion;
+ }
+
+ public void setHadoopBuildVersion(String hadoopBuildVersion) {
+ this.hadoopBuildVersion = hadoopBuildVersion;
+ }
+
+ @XmlElement(name = "hadoop-version-built-on")
+ public String getHadoopVersionBuiltOn() {
+ return hadoopVersionBuiltOn;
+ }
+
+ public void setHadoopVersionBuiltOn(String hadoopVersionBuiltOn) {
+ this.hadoopVersionBuiltOn = hadoopVersionBuiltOn;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f5fce6d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
index f198ba1..137b7c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
@@ -26,8 +26,11 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.YarnVersionInfo;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.codehaus.jackson.JsonGenerationException;
import org.codehaus.jackson.map.JsonMappingException;
@@ -88,6 +91,17 @@ public class TimelineUtils {
}
}
+ public static TimelineAbout createTimelineAbout(String about) {
+ TimelineAbout tsInfo = new TimelineAbout(about);
+ tsInfo.setHadoopBuildVersion(VersionInfo.getBuildVersion());
+ tsInfo.setHadoopVersion(VersionInfo.getVersion());
+ tsInfo.setHadoopVersionBuiltOn(VersionInfo.getDate());
+ tsInfo.setTimelineServiceBuildVersion(YarnVersionInfo.getBuildVersion());
+ tsInfo.setTimelineServiceVersion(YarnVersionInfo.getVersion());
+ tsInfo.setTimelineServiceVersionBuiltOn(YarnVersionInfo.getDate());
+ return tsInfo;
+ }
+
public static InetSocketAddress getTimelineTokenServiceAddress(
Configuration conf) {
InetSocketAddress timelineServiceAddr = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f5fce6d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java
index 4037f51..2e5e97c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSController.java
@@ -34,6 +34,10 @@ public class AHSController extends Controller {
setTitle("Application History");
}
+ public void about() {
+ render(AboutPage.class);
+ }
+
public void app() {
render(AppPage.class);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f5fce6d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java
index 80c4550..0193cbd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java
@@ -56,6 +56,7 @@ public class AHSWebApp extends WebApp implements YarnWebParams {
bind(ApplicationBaseProtocol.class).toInstance(historyClientService);
bind(TimelineDataManager.class).toInstance(timelineDataManager);
route("/", AHSController.class);
+ route("/about", AHSController.class, "about");
route(pajoin("/apps", APP_STATE), AHSController.class);
route(pajoin("/app", APPLICATION_ID), AHSController.class, "app");
route(pajoin("/appattempt", APPLICATION_ATTEMPT_ID), AHSController.class,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f5fce6d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
index 9edc9ab..e7a22bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
@@ -34,6 +34,7 @@ import javax.ws.rs.core.MediaType;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
import org.apache.hadoop.yarn.server.webapp.WebServices;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo;
@@ -41,6 +42,7 @@ import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppsInfo;
import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.hadoop.yarn.webapp.BadRequestException;
import com.google.inject.Inject;
@@ -56,6 +58,16 @@ public class AHSWebServices extends WebServices {
}
@GET
+ @Path("/about")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public TimelineAbout about(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res) {
+ init(res);
+ return TimelineUtils.createTimelineAbout("Generic History Service API");
+ }
+
+ @GET
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public AppsInfo get(@Context HttpServletRequest req,
@Context HttpServletResponse res) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f5fce6d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AboutBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AboutBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AboutBlock.java
new file mode 100644
index 0000000..b2419e9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AboutBlock.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
+
+import com.google.inject.Inject;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
+import org.apache.hadoop.yarn.util.YarnVersionInfo;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+import org.apache.hadoop.yarn.webapp.View;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.hadoop.yarn.webapp.view.InfoBlock;
+
+public class AboutBlock extends HtmlBlock {
+ @Inject
+ AboutBlock(View.ViewContext ctx) {
+ super(ctx);
+ }
+
+ @Override
+ protected void render(Block html) {
+ TimelineAbout tsInfo = TimelineUtils.createTimelineAbout(
+ "Timeline Server - Generic History Service UI");
+ info("Timeline Server Overview").
+ _("Timeline Server Version:", tsInfo.getTimelineServiceBuildVersion() +
+ " on " + tsInfo.getTimelineServiceVersionBuiltOn()).
+ _("Hadoop Version:", tsInfo.getHadoopBuildVersion() +
+ " on " + tsInfo.getHadoopVersionBuiltOn());
+ html._(InfoBlock.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f5fce6d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AboutPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AboutPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AboutPage.java
new file mode 100644
index 0000000..b50073a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AboutPage.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
+
+
+import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.hadoop.yarn.webapp.YarnWebParams;
+
+import static org.apache.hadoop.yarn.util.StringHelper.join;
+
+public class AboutPage extends AHSView {
+ @Override protected void preHead(Page.HTML<_> html) {
+ commonPreHead(html);
+ set(TITLE, "Timeline Server - Generic History Service");
+ }
+
+ @Override protected Class<? extends SubView> content() {
+ return AboutBlock.class;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f5fce6d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/NavBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/NavBlock.java
index 498503f..25ee4f0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/NavBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/NavBlock.java
@@ -43,6 +43,8 @@ public class NavBlock extends HtmlBlock {
div("#nav").
h3("Application History").
ul().
+ li().a(url("about"), "About").
+ _().
li().a(url("apps"), "Applications").
ul().
li().a(url("apps",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f5fce6d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
index 915e3f2..90f4d39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
@@ -42,17 +42,12 @@ import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
@@ -65,6 +60,9 @@ import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
import org.apache.hadoop.yarn.server.timeline.NameValuePair;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
+import org.apache.hadoop.yarn.util.YarnVersionInfo;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
@@ -86,43 +84,16 @@ public class TimelineWebServices {
this.timelineDataManager = timelineDataManager;
}
- @XmlRootElement(name = "about")
- @XmlAccessorType(XmlAccessType.NONE)
- @Public
- @Unstable
- public static class AboutInfo {
-
- private String about;
-
- public AboutInfo() {
-
- }
-
- public AboutInfo(String about) {
- this.about = about;
- }
-
- @XmlElement(name = "About")
- public String getAbout() {
- return about;
- }
-
- public void setAbout(String about) {
- this.about = about;
- }
-
- }
-
/**
* Return the description of the timeline web services.
*/
@GET
@Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
- public AboutInfo about(
+ public TimelineAbout about(
@Context HttpServletRequest req,
@Context HttpServletResponse res) {
init(res);
- return new AboutInfo("Timeline API");
+ return TimelineUtils.createTimelineAbout("Timeline API");
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f5fce6d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebApp.java
index 2cd7580..1e0886f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebApp.java
@@ -88,6 +88,20 @@ public class TestAHSWebApp extends ApplicationHistoryStoreTestUtils {
}
@Test
+ public void testAboutPage() throws Exception {
+ Injector injector =
+ WebAppTests.createMockInjector(ApplicationBaseProtocol.class,
+ mockApplicationHistoryClientService(0, 0, 0));
+ AboutPage aboutPageInstance = injector.getInstance(AboutPage.class);
+
+ aboutPageInstance.render();
+ WebAppTests.flushOutput(injector);
+
+ aboutPageInstance.render();
+ WebAppTests.flushOutput(injector);
+ }
+
+ @Test
public void testAppPage() throws Exception {
Injector injector =
WebAppTests.createMockInjector(ApplicationBaseProtocol.class,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f5fce6d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
index 913b80d..613df72 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
@@ -49,6 +49,8 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
@@ -57,6 +59,7 @@ import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -216,6 +219,34 @@ public class TestAHSWebServices extends JerseyTestBase {
}
@Test
+ public void testAbout() throws Exception {
+ WebResource r = resource();
+ ClientResponse response = r
+ .path("ws").path("v1").path("applicationhistory").path("about")
+ .queryParam("user.name", USERS[round])
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ TimelineAbout actualAbout = response.getEntity(TimelineAbout.class);
+ TimelineAbout expectedAbout =
+ TimelineUtils.createTimelineAbout("Generic History Service API");
+ Assert.assertNotNull(
+ "Timeline service about response is null", actualAbout);
+ Assert.assertEquals(expectedAbout.getAbout(), actualAbout.getAbout());
+ Assert.assertEquals(expectedAbout.getTimelineServiceVersion(),
+ actualAbout.getTimelineServiceVersion());
+ Assert.assertEquals(expectedAbout.getTimelineServiceBuildVersion(),
+ actualAbout.getTimelineServiceBuildVersion());
+ Assert.assertEquals(expectedAbout.getTimelineServiceVersionBuiltOn(),
+ actualAbout.getTimelineServiceVersionBuiltOn());
+ Assert.assertEquals(expectedAbout.getHadoopVersion(),
+ actualAbout.getHadoopVersion());
+ Assert.assertEquals(expectedAbout.getHadoopBuildVersion(),
+ actualAbout.getHadoopBuildVersion());
+ Assert.assertEquals(expectedAbout.getHadoopVersionBuiltOn(),
+ actualAbout.getHadoopVersionBuiltOn());
+ }
+
+ @Test
public void testAppsQuery() throws Exception {
WebResource r = resource();
ClientResponse response =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f5fce6d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java
index 7e96d2a..ab7cffd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java
@@ -29,7 +29,6 @@ import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -59,6 +58,8 @@ import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilter;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
@@ -184,10 +185,24 @@ public class TestTimelineWebServices extends JerseyTestBase {
.accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
- TimelineWebServices.AboutInfo about =
- response.getEntity(TimelineWebServices.AboutInfo.class);
- Assert.assertNotNull(about);
- Assert.assertEquals("Timeline API", about.getAbout());
+ TimelineAbout actualAbout = response.getEntity(TimelineAbout.class);
+ TimelineAbout expectedAbout =
+ TimelineUtils.createTimelineAbout("Timeline API");
+ Assert.assertNotNull(
+ "Timeline service about response is null", actualAbout);
+ Assert.assertEquals(expectedAbout.getAbout(), actualAbout.getAbout());
+ Assert.assertEquals(expectedAbout.getTimelineServiceVersion(),
+ actualAbout.getTimelineServiceVersion());
+ Assert.assertEquals(expectedAbout.getTimelineServiceBuildVersion(),
+ actualAbout.getTimelineServiceBuildVersion());
+ Assert.assertEquals(expectedAbout.getTimelineServiceVersionBuiltOn(),
+ actualAbout.getTimelineServiceVersionBuiltOn());
+ Assert.assertEquals(expectedAbout.getHadoopVersion(),
+ actualAbout.getHadoopVersion());
+ Assert.assertEquals(expectedAbout.getHadoopBuildVersion(),
+ actualAbout.getHadoopBuildVersion());
+ Assert.assertEquals(expectedAbout.getHadoopVersionBuiltOn(),
+ actualAbout.getHadoopVersionBuiltOn());
}
private static void verifyEntities(TimelineEntities entities) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f5fce6d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index acdd8ff..e177215 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -358,9 +358,17 @@ Here is a non-normative description of the API.
GET /ws/v1/timeline/
-Returns a JSON object describing the server instance.
+Returns a JSON object describing the server instance and version information.
- {"About":"Timeline API"}
+ {
+ About: "Timeline API",
+ timeline-service-version: "3.0.0-SNAPSHOT",
+ timeline-service-build-version: "3.0.0-SNAPSHOT from fcd0702c10ce574b887280476aba63d6682d5271 by zshen source checksum e9ec74ea3ff7bc9f3d35e9cac694fb",
+ timeline-service-version-built-on: "2015-05-13T19:45Z",
+ hadoop-version: "3.0.0-SNAPSHOT",
+ hadoop-build-version: "3.0.0-SNAPSHOT from fcd0702c10ce574b887280476aba63d6682d5271 by zshen source checksum 95874b192923b43cdb96a6e483afd60",
+ hadoop-version-built-on: "2015-05-13T19:44Z"
+ }
## <a name="REST_API_DOMAINS"></a>Domains `/ws/v1/timeline/domain`
@@ -889,6 +897,92 @@ Response Body:
Users can access the generic historic information of applications via REST
APIs.
+## <a name="REST_API_ABOUT"></a>About
+
+With the about API, you can get an timeline about resource that contains
+generic history REST API description and version information.
+
+It is essentially a XML/JSON-serialized form of the YARN `TimelineAbout`
+structure.
+
+### URI:
+
+Use the following URI to obtain an timeline about object.
+
+ http(s)://<timeline server http(s) address:port>/ws/v1/applicationhistory/about
+
+### HTTP Operations Supported:
+
+ GET
+
+### Query Parameters Supported:
+
+None
+
+### Elements of the `about` (Application) Object:
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| `About` | string | The description about the service |
+| `timeline-service-version` | string | The timeline service version |
+| `timeline-service-build-version` | string | The timeline service build version |
+| `timeline-service-version-built-on` | string | On what time the timeline service is built |
+| `hadoop-version` | string | Hadoop version |
+| `hadoop-build-version` | string | Hadoop build version |
+| `hadoop-version-built-on` | string | On what time Hadoop is built |
+
+### Response Examples:
+
+#### JSON response
+
+HTTP Request:
+
+ http://localhost:8188/ws/v1/applicationhistory/about
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+
+Response Body:
+
+ {
+ About: "Generic History Service API",
+ timeline-service-version: "3.0.0-SNAPSHOT",
+ timeline-service-build-version: "3.0.0-SNAPSHOT from fcd0702c10ce574b887280476aba63d6682d5271 by zshen source checksum e9ec74ea3ff7bc9f3d35e9cac694fb",
+ timeline-service-version-built-on: "2015-05-13T19:45Z",
+ hadoop-version: "3.0.0-SNAPSHOT",
+ hadoop-build-version: "3.0.0-SNAPSHOT from fcd0702c10ce574b887280476aba63d6682d5271 by zshen source checksum 95874b192923b43cdb96a6e483afd60",
+ hadoop-version-built-on: "2015-05-13T19:44Z"
+ }
+
+#### XML response
+
+HTTP Request:
+
+ GET http://localhost:8188/ws/v1/applicationhistory/about
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 748
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <about>
+ <About>Generic History Service API</About>
+ <hadoop-build-version>3.0.0-SNAPSHOT from fcd0702c10ce574b887280476aba63d6682d5271 by zshen source checksum 95874b192923b43cdb96a6e483afd60</hadoop-build-version>
+ <hadoop-version>3.0.0-SNAPSHOT</hadoop-version>
+ <hadoop-version-built-on>2015-05-13T19:44Z</hadoop-version-built-on>
+ <timeline-service-build-version>3.0.0-SNAPSHOT from fcd0702c10ce574b887280476aba63d6682d5271 by zshen source checksum e9ec74ea3ff7bc9f3d35e9cac694fb</timeline-service-build-version>
+ <timeline-service-version>3.0.0-SNAPSHOT</timeline-service-version>
+ <timeline-service-version-built-on>2015-05-13T19:45Z</timeline-service-version-built-on>
+ </about>
+
## <a name="REST_API_LIST_APPLICATIONS"></a>Application List
With the Application List API, you can obtain a collection of resources, each
@@ -1129,7 +1223,8 @@ With the Application API, you can get an application resource contains
information about a particular application that was running on an YARN
cluster.
-It is essentially a JSON-serialized form of the YARN `ApplicationReport` structure.
+It is essentially a XML/JSON-serialized form of the YARN `ApplicationReport`
+structure.
### URI:
[47/50] [abbrv] hadoop git commit: YARN-3646. Applications are
getting stuck some times in case of retry policy forever. Contributed by Raju
Bairishetti.
Posted by zj...@apache.org.
YARN-3646. Applications are getting stuck some times in case of retry
policy forever. Contributed by Raju Bairishetti.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/369d7c47
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/369d7c47
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/369d7c47
Branch: refs/heads/YARN-2928
Commit: 369d7c4766341ab6391d649ab2d7abfe46c11280
Parents: 72ff460
Author: Devaraj K <de...@apache.org>
Authored: Thu May 21 20:14:44 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 12:42:58 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../yarn/client/api/impl/TestYarnClient.java | 32 ++++++++++++++++++++
.../org/apache/hadoop/yarn/client/RMProxy.java | 15 ++++-----
3 files changed, 43 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/369d7c47/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cb3ecb7..9ec9618 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -619,6 +619,9 @@ Release 2.7.1 - UNRELEASED
YARN-3694. Fix dead link for TimelineServer REST API.
(Jagadesh Kiran N via aajisaka)
+ YARN-3646. Applications are getting stuck some times in case of retry
+ policy forever. (Raju Bairishetti via devaraj)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/369d7c47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index 511fa4a..bc40b9a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -1265,4 +1265,36 @@ public class TestYarnClient {
ReservationSystemTestUtil.reservationQ);
return request;
}
+
+ @Test(timeout = 30000, expected = ApplicationNotFoundException.class)
+ public void testShouldNotRetryForeverForNonNetworkExceptions() throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, -1);
+
+ ResourceManager rm = null;
+ YarnClient yarnClient = null;
+ try {
+ // start rm
+ rm = new ResourceManager();
+ rm.init(conf);
+ rm.start();
+
+ yarnClient = YarnClient.createYarnClient();
+ yarnClient.init(conf);
+ yarnClient.start();
+
+ // create invalid application id
+ ApplicationId appId = ApplicationId.newInstance(1430126768L, 10645);
+
+ // RM should throw ApplicationNotFoundException exception
+ yarnClient.getApplicationReport(appId);
+ } finally {
+ if (yarnClient != null) {
+ yarnClient.stop();
+ }
+ if (rm != null) {
+ rm.stop();
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/369d7c47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
index fa8d642..28628f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
@@ -224,19 +224,20 @@ public class RMProxy<T> {
failoverSleepBaseMs, failoverSleepMaxMs);
}
- if (waitForEver) {
- return RetryPolicies.RETRY_FOREVER;
- }
-
if (rmConnectionRetryIntervalMS < 0) {
throw new YarnRuntimeException("Invalid Configuration. " +
YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS +
" should not be negative.");
}
- RetryPolicy retryPolicy =
- RetryPolicies.retryUpToMaximumTimeWithFixedSleep(rmConnectWaitMS,
- rmConnectionRetryIntervalMS, TimeUnit.MILLISECONDS);
+ RetryPolicy retryPolicy = null;
+ if (waitForEver) {
+ retryPolicy = RetryPolicies.RETRY_FOREVER;
+ } else {
+ retryPolicy =
+ RetryPolicies.retryUpToMaximumTimeWithFixedSleep(rmConnectWaitMS,
+ rmConnectionRetryIntervalMS, TimeUnit.MILLISECONDS);
+ }
Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
new HashMap<Class<? extends Exception>, RetryPolicy>();
[03/50] [abbrv] hadoop git commit: HDFS-8397. Refactor the error
handling code in DataStreamer. Contributed by Tsz Wo Nicholas Sze.
Posted by zj...@apache.org.
HDFS-8397. Refactor the error handling code in DataStreamer. Contributed by Tsz Wo Nicholas Sze.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c78b397b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c78b397b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c78b397b
Branch: refs/heads/YARN-2928
Commit: c78b397ba7509e2baf7cba8fef40016e4f2819f2
Parents: 26eb1b4
Author: Jing Zhao <ji...@apache.org>
Authored: Fri May 15 16:14:54 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:47:00 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../org/apache/hadoop/hdfs/DataStreamer.java | 543 +++++++++++--------
2 files changed, 308 insertions(+), 238 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c78b397b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 6c0923c..35e81f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -554,6 +554,9 @@ Release 2.8.0 - UNRELEASED
HDFS-6888. Allow selectively audit logging ops (Chen He via vinayakumarb)
+ HDFS-8397. Refactor the error handling code in DataStreamer.
+ (Tsz Wo Nicholas Sze via jing9)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c78b397b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index b472b8c..cecd5a0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -38,7 +38,6 @@ import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.logging.Log;
@@ -46,6 +45,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.BlockWrite;
import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
@@ -208,6 +208,133 @@ class DataStreamer extends Daemon {
}
}
+ static class ErrorState {
+ private boolean error = false;
+ private int badNodeIndex = -1;
+ private int restartingNodeIndex = -1;
+ private long restartingNodeDeadline = 0;
+ private final long datanodeRestartTimeout;
+
+ ErrorState(long datanodeRestartTimeout) {
+ this.datanodeRestartTimeout = datanodeRestartTimeout;
+ }
+
+ synchronized void reset() {
+ error = false;
+ badNodeIndex = -1;
+ restartingNodeIndex = -1;
+ restartingNodeDeadline = 0;
+ }
+
+ synchronized boolean hasError() {
+ return error;
+ }
+
+ synchronized boolean hasDatanodeError() {
+ return error && isNodeMarked();
+ }
+
+ synchronized void setError(boolean err) {
+ this.error = err;
+ }
+
+ synchronized void setBadNodeIndex(int index) {
+ this.badNodeIndex = index;
+ }
+
+ synchronized int getBadNodeIndex() {
+ return badNodeIndex;
+ }
+
+ synchronized int getRestartingNodeIndex() {
+ return restartingNodeIndex;
+ }
+
+ synchronized void initRestartingNode(int i, String message) {
+ restartingNodeIndex = i;
+ restartingNodeDeadline = Time.monotonicNow() + datanodeRestartTimeout;
+ // If the data streamer has already set the primary node
+ // bad, clear it. It is likely that the write failed due to
+ // the DN shutdown. Even if it was a real failure, the pipeline
+ // recovery will take care of it.
+ badNodeIndex = -1;
+ LOG.info(message);
+ }
+
+ synchronized boolean isRestartingNode() {
+ return restartingNodeIndex >= 0;
+ }
+
+ synchronized boolean isNodeMarked() {
+ return badNodeIndex >= 0 || isRestartingNode();
+ }
+
+ /**
+ * This method is used when no explicit error report was received, but
+ * something failed. The first node is a suspect or unsure about the cause
+ * so that it is marked as failed.
+ */
+ synchronized void markFirstNodeIfNotMarked() {
+ // There should be no existing error and no ongoing restart.
+ if (!isNodeMarked()) {
+ badNodeIndex = 0;
+ }
+ }
+
+ synchronized void adjustState4RestartingNode() {
+ // Just took care of a node error while waiting for a node restart
+ if (restartingNodeIndex >= 0) {
+ // If the error came from a node further away than the restarting
+ // node, the restart must have been complete.
+ if (badNodeIndex > restartingNodeIndex) {
+ restartingNodeIndex = -1;
+ } else if (badNodeIndex < restartingNodeIndex) {
+ // the node index has shifted.
+ restartingNodeIndex--;
+ } else {
+ throw new IllegalStateException("badNodeIndex = " + badNodeIndex
+ + " = restartingNodeIndex = " + restartingNodeIndex);
+ }
+ }
+
+ if (!isRestartingNode()) {
+ error = false;
+ }
+ badNodeIndex = -1;
+ }
+
+ synchronized void checkRestartingNodeDeadline(DatanodeInfo[] nodes) {
+ if (restartingNodeIndex >= 0) {
+ if (!error) {
+ throw new IllegalStateException("error=false while checking" +
+ " restarting node deadline");
+ }
+
+ // check badNodeIndex
+ if (badNodeIndex == restartingNodeIndex) {
+ // ignore, if came from the restarting node
+ badNodeIndex = -1;
+ }
+ // not within the deadline
+ if (Time.monotonicNow() >= restartingNodeDeadline) {
+ // expired. declare the restarting node dead
+ restartingNodeDeadline = 0;
+ final int i = restartingNodeIndex;
+ restartingNodeIndex = -1;
+ LOG.warn("Datanode " + i + " did not restart within "
+ + datanodeRestartTimeout + "ms: " + nodes[i]);
+ // Mark the restarting node as failed. If there is any other failed
+ // node during the last pipeline construction attempt, it will not be
+ // overwritten/dropped. In this case, the restarting node will get
+ // excluded in the following attempt, if it still does not come up.
+ if (badNodeIndex == -1) {
+ badNodeIndex = i;
+ }
+ }
+ }
+ }
+ }
+
private volatile boolean streamerClosed = false;
private ExtendedBlock block; // its length is number of bytes acked
private Token<BlockTokenIdentifier> accessToken;
@@ -217,11 +344,8 @@ class DataStreamer extends Daemon {
private volatile DatanodeInfo[] nodes = null; // list of targets for current block
private volatile StorageType[] storageTypes = null;
private volatile String[] storageIDs = null;
- volatile boolean hasError = false;
- volatile int errorIndex = -1;
- // Restarting node index
- AtomicInteger restartingNodeIndex = new AtomicInteger(-1);
- private long restartDeadline = 0; // Deadline of DN restart
+ private final ErrorState errorState;
+
private BlockConstructionStage stage; // block construction stage
private long bytesSent = 0; // number of bytes that've been sent
private final boolean isLazyPersistFile;
@@ -287,11 +411,13 @@ class DataStreamer extends Daemon {
this.cachingStrategy = cachingStrategy;
this.byteArrayManager = byteArrayManage;
this.isLazyPersistFile = isLazyPersist(stat);
- this.dfsclientSlowLogThresholdMs =
- dfsClient.getConf().getSlowIoWarningThresholdMs();
- this.excludedNodes = initExcludedNodes();
this.isAppend = isAppend;
this.favoredNodes = favoredNodes;
+
+ final DfsClientConf conf = dfsClient.getConf();
+ this.dfsclientSlowLogThresholdMs = conf.getSlowIoWarningThresholdMs();
+ this.excludedNodes = initExcludedNodes(conf.getExcludedNodesCacheExpiry());
+ this.errorState = new ErrorState(conf.getDatanodeRestartTimeout());
}
/**
@@ -334,7 +460,6 @@ class DataStreamer extends Daemon {
void setPipelineInConstruction(LocatedBlock lastBlock) throws IOException{
// setup pipeline to append to the last block XXX retries??
setPipeline(lastBlock);
- errorIndex = -1; // no errors yet.
if (nodes.length < 1) {
throw new IOException("Unable to retrieve blocks locations " +
" for last block " + block +
@@ -375,6 +500,10 @@ class DataStreamer extends Daemon {
stage = BlockConstructionStage.PIPELINE_SETUP_CREATE;
}
+ private boolean shouldStop() {
+ return streamerClosed || errorState.hasError() || !dfsClient.clientRunning;
+ }
+
/*
* streamer thread is the only thread that opens streams to datanode,
* and closes them. Any error recovery is also done by this thread.
@@ -385,7 +514,7 @@ class DataStreamer extends Daemon {
TraceScope scope = NullScope.INSTANCE;
while (!streamerClosed && dfsClient.clientRunning) {
// if the Responder encountered an error, shutdown Responder
- if (hasError && response != null) {
+ if (errorState.hasError() && response != null) {
try {
response.close();
response.join();
@@ -398,17 +527,13 @@ class DataStreamer extends Daemon {
DFSPacket one;
try {
// process datanode IO errors if any
- boolean doSleep = false;
- if (hasError && (errorIndex >= 0 || restartingNodeIndex.get() >= 0)) {
- doSleep = processDatanodeError();
- }
+ boolean doSleep = processDatanodeError();
final int halfSocketTimeout = dfsClient.getConf().getSocketTimeout()/2;
synchronized (dataQueue) {
// wait for a packet to be sent.
long now = Time.monotonicNow();
- while ((!streamerClosed && !hasError && dfsClient.clientRunning
- && dataQueue.size() == 0 &&
+ while ((!shouldStop() && dataQueue.size() == 0 &&
(stage != BlockConstructionStage.DATA_STREAMING ||
stage == BlockConstructionStage.DATA_STREAMING &&
now - lastPacket < halfSocketTimeout)) || doSleep ) {
@@ -424,13 +549,12 @@ class DataStreamer extends Daemon {
doSleep = false;
now = Time.monotonicNow();
}
- if (streamerClosed || hasError || !dfsClient.clientRunning) {
+ if (shouldStop()) {
continue;
}
// get packet to be sent.
if (dataQueue.isEmpty()) {
one = createHeartbeatPacket();
- assert one != null;
} else {
try {
backOffIfNecessary();
@@ -460,7 +584,7 @@ class DataStreamer extends Daemon {
LOG.debug("Append to block " + block);
}
setupPipelineForAppendOrRecovery();
- if (true == streamerClosed) {
+ if (streamerClosed) {
continue;
}
initDataStreaming();
@@ -478,8 +602,7 @@ class DataStreamer extends Daemon {
if (one.isLastPacketInBlock()) {
// wait for all data packets have been successfully acked
synchronized (dataQueue) {
- while (!streamerClosed && !hasError &&
- ackQueue.size() != 0 && dfsClient.clientRunning) {
+ while (!shouldStop() && ackQueue.size() != 0) {
try {
// wait for acks to arrive from datanodes
dataQueue.wait(1000);
@@ -488,7 +611,7 @@ class DataStreamer extends Daemon {
}
}
}
- if (streamerClosed || hasError || !dfsClient.clientRunning) {
+ if (shouldStop()) {
continue;
}
stage = BlockConstructionStage.PIPELINE_CLOSE;
@@ -524,7 +647,7 @@ class DataStreamer extends Daemon {
// effect. Pipeline recovery can handle only one node error at a
// time. If the primary node fails again during the recovery, it
// will be taken out then.
- tryMarkPrimaryDatanodeFailed();
+ errorState.markFirstNodeIfNotMarked();
throw e;
} finally {
writeScope.close();
@@ -537,7 +660,7 @@ class DataStreamer extends Daemon {
bytesSent = tmpBytesSent;
}
- if (streamerClosed || hasError || !dfsClient.clientRunning) {
+ if (shouldStop()) {
continue;
}
@@ -545,12 +668,11 @@ class DataStreamer extends Daemon {
if (one.isLastPacketInBlock()) {
// wait for the close packet has been acked
synchronized (dataQueue) {
- while (!streamerClosed && !hasError &&
- ackQueue.size() != 0 && dfsClient.clientRunning) {
+ while (!shouldStop() && ackQueue.size() != 0) {
dataQueue.wait(1000);// wait for acks to arrive from datanodes
}
}
- if (streamerClosed || hasError || !dfsClient.clientRunning) {
+ if (shouldStop()) {
continue;
}
@@ -564,7 +686,7 @@ class DataStreamer extends Daemon {
}
} catch (Throwable e) {
// Log warning if there was a real error.
- if (restartingNodeIndex.get() == -1) {
+ if (!errorState.isRestartingNode()) {
// Since their messages are descriptive enough, do not always
// log a verbose stack-trace WARN for quota exceptions.
if (e instanceof QuotaExceededException) {
@@ -575,8 +697,8 @@ class DataStreamer extends Daemon {
}
lastException.set(e);
assert !(e instanceof NullPointerException);
- hasError = true;
- if (errorIndex == -1 && restartingNodeIndex.get() == -1) {
+ errorState.setError(true);
+ if (!errorState.isNodeMarked()) {
// Not a datanode issue
streamerClosed = true;
}
@@ -773,40 +895,6 @@ class DataStreamer extends Daemon {
}
}
- // The following synchronized methods are used whenever
- // errorIndex or restartingNodeIndex is set. This is because
- // check & set needs to be atomic. Simply reading variables
- // does not require a synchronization. When responder is
- // not running (e.g. during pipeline recovery), there is no
- // need to use these methods.
-
- /** Set the error node index. Called by responder */
- synchronized void setErrorIndex(int idx) {
- errorIndex = idx;
- }
-
- /** Set the restarting node index. Called by responder */
- synchronized void setRestartingNodeIndex(int idx) {
- restartingNodeIndex.set(idx);
- // If the data streamer has already set the primary node
- // bad, clear it. It is likely that the write failed due to
- // the DN shutdown. Even if it was a real failure, the pipeline
- // recovery will take care of it.
- errorIndex = -1;
- }
-
- /**
- * This method is used when no explicit error report was received,
- * but something failed. When the primary node is a suspect or
- * unsure about the cause, the primary node is marked as failed.
- */
- synchronized void tryMarkPrimaryDatanodeFailed() {
- // There should be no existing error and no ongoing restart.
- if ((errorIndex == -1) && (restartingNodeIndex.get() == -1)) {
- errorIndex = 0;
- }
- }
-
/**
* Examine whether it is worth waiting for a node to restart.
* @param index the node index
@@ -883,20 +971,16 @@ class DataStreamer extends Daemon {
// the local node or the only one in the pipeline.
if (PipelineAck.isRestartOOBStatus(reply) &&
shouldWaitForRestart(i)) {
- restartDeadline = dfsClient.getConf().getDatanodeRestartTimeout()
- + Time.monotonicNow();
- setRestartingNodeIndex(i);
- String message = "A datanode is restarting: " + targets[i];
- LOG.info(message);
+ final String message = "Datanode " + i + " is restarting: "
+ + targets[i];
+ errorState.initRestartingNode(i, message);
throw new IOException(message);
}
// node error
if (reply != SUCCESS) {
- setErrorIndex(i); // first bad datanode
+ errorState.setBadNodeIndex(i); // mark bad datanode
throw new IOException("Bad response " + reply +
- " for block " + block +
- " from datanode " +
- targets[i]);
+ " for " + block + " from datanode " + targets[i]);
}
}
@@ -954,14 +1038,12 @@ class DataStreamer extends Daemon {
} catch (Exception e) {
if (!responderClosed) {
lastException.set(e);
- hasError = true;
- // If no explicit error report was received, mark the primary
- // node as failed.
- tryMarkPrimaryDatanodeFailed();
+ errorState.setError(true);
+ errorState.markFirstNodeIfNotMarked();
synchronized (dataQueue) {
dataQueue.notifyAll();
}
- if (restartingNodeIndex.get() == -1) {
+ if (!errorState.isRestartingNode()) {
LOG.warn("Exception for " + block, e);
}
responderClosed = true;
@@ -978,11 +1060,16 @@ class DataStreamer extends Daemon {
}
}
- // If this stream has encountered any errors so far, shutdown
- // threads and mark stream as closed. Returns true if we should
- // sleep for a while after returning from this call.
- //
+ /**
+ * If this stream has encountered any errors, shutdown threads
+ * and mark the stream as closed.
+ *
+ * @return true if it should sleep for a while after returning.
+ */
private boolean processDatanodeError() throws IOException {
+ if (!errorState.hasDatanodeError()) {
+ return false;
+ }
if (response != null) {
LOG.info("Error Recovery for " + block +
" waiting for responder to exit. ");
@@ -1064,7 +1151,7 @@ class DataStreamer extends Daemon {
.append("The current failed datanode replacement policy is ")
.append(dfsClient.dtpReplaceDatanodeOnFailure).append(", and ")
.append("a client may configure this via '")
- .append(HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.POLICY_KEY)
+ .append(BlockWrite.ReplaceDatanodeOnFailure.POLICY_KEY)
.append("' in its configuration.")
.toString());
}
@@ -1190,157 +1277,141 @@ class DataStreamer extends Daemon {
boolean success = false;
long newGS = 0L;
while (!success && !streamerClosed && dfsClient.clientRunning) {
- // Sleep before reconnect if a dn is restarting.
- // This process will be repeated until the deadline or the datanode
- // starts back up.
- if (restartingNodeIndex.get() >= 0) {
- // 4 seconds or the configured deadline period, whichever is shorter.
- // This is the retry interval and recovery will be retried in this
- // interval until timeout or success.
- long delay = Math.min(dfsClient.getConf().getDatanodeRestartTimeout(),
- 4000L);
- try {
- Thread.sleep(delay);
- } catch (InterruptedException ie) {
- lastException.set(new IOException("Interrupted while waiting for " +
- "datanode to restart. " + nodes[restartingNodeIndex.get()]));
- streamerClosed = true;
- return false;
- }
+ if (!handleRestartingDatanode()) {
+ return false;
}
- boolean isRecovery = hasError;
- // remove bad datanode from list of datanodes.
- // If errorIndex was not set (i.e. appends), then do not remove
- // any datanodes
- //
- if (errorIndex >= 0) {
- StringBuilder pipelineMsg = new StringBuilder();
- for (int j = 0; j < nodes.length; j++) {
- pipelineMsg.append(nodes[j]);
- if (j < nodes.length - 1) {
- pipelineMsg.append(", ");
- }
- }
- if (nodes.length <= 1) {
- lastException.set(new IOException("All datanodes " + pipelineMsg
- + " are bad. Aborting..."));
- streamerClosed = true;
- return false;
- }
- LOG.warn("Error Recovery for block " + block +
- " in pipeline " + pipelineMsg +
- ": bad datanode " + nodes[errorIndex]);
- failed.add(nodes[errorIndex]);
-
- DatanodeInfo[] newnodes = new DatanodeInfo[nodes.length-1];
- arraycopy(nodes, newnodes, errorIndex);
-
- final StorageType[] newStorageTypes = new StorageType[newnodes.length];
- arraycopy(storageTypes, newStorageTypes, errorIndex);
-
- final String[] newStorageIDs = new String[newnodes.length];
- arraycopy(storageIDs, newStorageIDs, errorIndex);
-
- setPipeline(newnodes, newStorageTypes, newStorageIDs);
-
- // Just took care of a node error while waiting for a node restart
- if (restartingNodeIndex.get() >= 0) {
- // If the error came from a node further away than the restarting
- // node, the restart must have been complete.
- if (errorIndex > restartingNodeIndex.get()) {
- restartingNodeIndex.set(-1);
- } else if (errorIndex < restartingNodeIndex.get()) {
- // the node index has shifted.
- restartingNodeIndex.decrementAndGet();
- } else {
- // this shouldn't happen...
- assert false;
- }
- }
- if (restartingNodeIndex.get() == -1) {
- hasError = false;
- }
- lastException.clear();
- errorIndex = -1;
+ final boolean isRecovery = errorState.hasError();
+ if (!handleBadDatanode()) {
+ return false;
}
- // Check if replace-datanode policy is satisfied.
- if (dfsClient.dtpReplaceDatanodeOnFailure.satisfy(stat.getReplication(),
- nodes, isAppend, isHflushed)) {
- try {
- addDatanode2ExistingPipeline();
- } catch(IOException ioe) {
- if (!dfsClient.dtpReplaceDatanodeOnFailure.isBestEffort()) {
- throw ioe;
- }
- LOG.warn("Failed to replace datanode."
- + " Continue with the remaining datanodes since "
- + HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.BEST_EFFORT_KEY
- + " is set to true.", ioe);
- }
- }
+ handleDatanodeReplacement();
// get a new generation stamp and an access token
- LocatedBlock lb = dfsClient.namenode.updateBlockForPipeline(block, dfsClient.clientName);
+ final LocatedBlock lb = updateBlockForPipeline();
newGS = lb.getBlock().getGenerationStamp();
accessToken = lb.getBlockToken();
// set up the pipeline again with the remaining nodes
- if (failPacket) { // for testing
- success = createBlockOutputStream(nodes, storageTypes, newGS, isRecovery);
- failPacket = false;
- try {
- // Give DNs time to send in bad reports. In real situations,
- // good reports should follow bad ones, if client committed
- // with those nodes.
- Thread.sleep(2000);
- } catch (InterruptedException ie) {}
- } else {
- success = createBlockOutputStream(nodes, storageTypes, newGS, isRecovery);
- }
+ success = createBlockOutputStream(nodes, storageTypes, newGS, isRecovery);
- if (restartingNodeIndex.get() >= 0) {
- assert hasError == true;
- // check errorIndex set above
- if (errorIndex == restartingNodeIndex.get()) {
- // ignore, if came from the restarting node
- errorIndex = -1;
- }
- // still within the deadline
- if (Time.monotonicNow() < restartDeadline) {
- continue; // with in the deadline
- }
- // expired. declare the restarting node dead
- restartDeadline = 0;
- int expiredNodeIndex = restartingNodeIndex.get();
- restartingNodeIndex.set(-1);
- LOG.warn("Datanode did not restart in time: " +
- nodes[expiredNodeIndex]);
- // Mark the restarting node as failed. If there is any other failed
- // node during the last pipeline construction attempt, it will not be
- // overwritten/dropped. In this case, the restarting node will get
- // excluded in the following attempt, if it still does not come up.
- if (errorIndex == -1) {
- errorIndex = expiredNodeIndex;
- }
- // From this point on, normal pipeline recovery applies.
- }
+ failPacket4Testing();
+
+ errorState.checkRestartingNodeDeadline(nodes);
} // while
if (success) {
- // update pipeline at the namenode
- ExtendedBlock newBlock = new ExtendedBlock(
- block.getBlockPoolId(), block.getBlockId(), block.getNumBytes(), newGS);
- dfsClient.namenode.updatePipeline(dfsClient.clientName, block, newBlock,
- nodes, storageIDs);
- // update client side generation stamp
- block = newBlock;
+ block = updatePipeline(newGS);
}
return false; // do not sleep, continue processing
}
/**
+ * Sleep if a node is restarting.
+ * This process is repeated until the deadline or the node starts back up.
+ * @return true if it should continue.
+ */
+ private boolean handleRestartingDatanode() {
+ if (errorState.isRestartingNode()) {
+ // 4 seconds or the configured deadline period, whichever is shorter.
+ // This is the retry interval and recovery will be retried in this
+ // interval until timeout or success.
+ final long delay = Math.min(errorState.datanodeRestartTimeout, 4000L);
+ try {
+ Thread.sleep(delay);
+ } catch (InterruptedException ie) {
+ lastException.set(new IOException(
+ "Interrupted while waiting for restarting "
+ + nodes[errorState.getRestartingNodeIndex()]));
+ streamerClosed = true;
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Remove bad node from list of nodes if badNodeIndex was set.
+ * @return true if it should continue.
+ */
+ private boolean handleBadDatanode() {
+ final int badNodeIndex = errorState.getBadNodeIndex();
+ if (badNodeIndex >= 0) {
+ if (nodes.length <= 1) {
+ lastException.set(new IOException("All datanodes "
+ + Arrays.toString(nodes) + " are bad. Aborting..."));
+ streamerClosed = true;
+ return false;
+ }
+
+ LOG.warn("Error Recovery for " + block + " in pipeline "
+ + Arrays.toString(nodes) + ": datanode " + badNodeIndex
+ + "("+ nodes[badNodeIndex] + ") is bad.");
+ failed.add(nodes[badNodeIndex]);
+
+ DatanodeInfo[] newnodes = new DatanodeInfo[nodes.length-1];
+ arraycopy(nodes, newnodes, badNodeIndex);
+
+ final StorageType[] newStorageTypes = new StorageType[newnodes.length];
+ arraycopy(storageTypes, newStorageTypes, badNodeIndex);
+
+ final String[] newStorageIDs = new String[newnodes.length];
+ arraycopy(storageIDs, newStorageIDs, badNodeIndex);
+
+ setPipeline(newnodes, newStorageTypes, newStorageIDs);
+
+ errorState.adjustState4RestartingNode();
+ lastException.clear();
+ }
+ return true;
+ }
+
+ /** Add a datanode if replace-datanode policy is satisfied. */
+ private void handleDatanodeReplacement() throws IOException {
+ if (dfsClient.dtpReplaceDatanodeOnFailure.satisfy(stat.getReplication(),
+ nodes, isAppend, isHflushed)) {
+ try {
+ addDatanode2ExistingPipeline();
+ } catch(IOException ioe) {
+ if (!dfsClient.dtpReplaceDatanodeOnFailure.isBestEffort()) {
+ throw ioe;
+ }
+ LOG.warn("Failed to replace datanode."
+ + " Continue with the remaining datanodes since "
+ + BlockWrite.ReplaceDatanodeOnFailure.BEST_EFFORT_KEY
+ + " is set to true.", ioe);
+ }
+ }
+ }
+
+ private void failPacket4Testing() {
+ if (failPacket) { // for testing
+ failPacket = false;
+ try {
+ // Give DNs time to send in bad reports. In real situations,
+ // good reports should follow bad ones, if client committed
+ // with those nodes.
+ Thread.sleep(2000);
+ } catch (InterruptedException ie) {}
+ }
+ }
+
+ LocatedBlock updateBlockForPipeline() throws IOException {
+ return dfsClient.namenode.updateBlockForPipeline(
+ block, dfsClient.clientName);
+ }
+
+ /** update pipeline at the namenode */
+ ExtendedBlock updatePipeline(long newGS) throws IOException {
+ final ExtendedBlock newBlock = new ExtendedBlock(
+ block.getBlockPoolId(), block.getBlockId(), block.getNumBytes(), newGS);
+ dfsClient.namenode.updatePipeline(dfsClient.clientName, block, newBlock,
+ nodes, storageIDs);
+ return newBlock;
+ }
+
+ /**
* Open a DataStreamer to a DataNode so that it can be written to.
* This happens when a file is created and each time a new block is allocated.
* Must get block ID and the IDs of the destinations from the namenode.
@@ -1354,9 +1425,8 @@ class DataStreamer extends Daemon {
boolean success = false;
ExtendedBlock oldBlock = block;
do {
- hasError = false;
+ errorState.reset();
lastException.clear();
- errorIndex = -1;
success = false;
DatanodeInfo[] excluded =
@@ -1382,8 +1452,9 @@ class DataStreamer extends Daemon {
dfsClient.namenode.abandonBlock(block, stat.getFileId(), src,
dfsClient.clientName);
block = null;
- LOG.info("Excluding datanode " + nodes[errorIndex]);
- excludedNodes.put(nodes[errorIndex], nodes[errorIndex]);
+ final DatanodeInfo badNode = nodes[errorState.getBadNodeIndex()];
+ LOG.info("Excluding datanode " + badNode);
+ excludedNodes.put(badNode, badNode);
}
} while (!success && --count >= 0);
@@ -1464,7 +1535,7 @@ class DataStreamer extends Daemon {
// from the local datanode. Thus it is safe to treat this as a
// regular node error.
if (PipelineAck.isRestartOOBStatus(pipelineStatus) &&
- restartingNodeIndex.get() == -1) {
+ !errorState.isRestartingNode()) {
checkRestart = true;
throw new IOException("A datanode is restarting.");
}
@@ -1475,10 +1546,9 @@ class DataStreamer extends Daemon {
assert null == blockStream : "Previous blockStream unclosed";
blockStream = out;
result = true; // success
- restartingNodeIndex.set(-1);
- hasError = false;
+ errorState.reset();
} catch (IOException ie) {
- if (restartingNodeIndex.get() == -1) {
+ if (!errorState.isRestartingNode()) {
LOG.info("Exception in createBlockOutputStream", ie);
}
if (ie instanceof InvalidEncryptionKeyException && refetchEncryptionKey > 0) {
@@ -1498,24 +1568,21 @@ class DataStreamer extends Daemon {
for (int i = 0; i < nodes.length; i++) {
// NB: Unconditionally using the xfer addr w/o hostname
if (firstBadLink.equals(nodes[i].getXferAddr())) {
- errorIndex = i;
+ errorState.setBadNodeIndex(i);
break;
}
}
} else {
assert checkRestart == false;
- errorIndex = 0;
+ errorState.setBadNodeIndex(0);
}
+
+ final int i = errorState.getBadNodeIndex();
// Check whether there is a restart worth waiting for.
- if (checkRestart && shouldWaitForRestart(errorIndex)) {
- restartDeadline = dfsClient.getConf().getDatanodeRestartTimeout()
- + Time.monotonicNow();
- restartingNodeIndex.set(errorIndex);
- errorIndex = -1;
- LOG.info("Waiting for the datanode to be restarted: " +
- nodes[restartingNodeIndex.get()]);
+ if (checkRestart && shouldWaitForRestart(i)) {
+ errorState.initRestartingNode(i, "Datanode " + i + " is restarting: " + nodes[i]);
}
- hasError = true;
+ errorState.setError(true);
lastException.set(ie);
result = false; // error
} finally {
@@ -1699,10 +1766,10 @@ class DataStreamer extends Daemon {
return new DFSPacket(buf, 0, 0, DFSPacket.HEART_BEAT_SEQNO, 0, false);
}
- private LoadingCache<DatanodeInfo, DatanodeInfo> initExcludedNodes() {
- return CacheBuilder.newBuilder().expireAfterWrite(
- dfsClient.getConf().getExcludedNodesCacheExpiry(),
- TimeUnit.MILLISECONDS)
+ private static LoadingCache<DatanodeInfo, DatanodeInfo> initExcludedNodes(
+ long excludedNodesCacheExpiry) {
+ return CacheBuilder.newBuilder()
+ .expireAfterWrite(excludedNodesCacheExpiry, TimeUnit.MILLISECONDS)
.removalListener(new RemovalListener<DatanodeInfo, DatanodeInfo>() {
@Override
public void onRemoval(
[33/50] [abbrv] hadoop git commit: YARN-2821. Fixed a problem that
DistributedShell AM may hang if restarted. Contributed by Varun Vasudev
Posted by zj...@apache.org.
YARN-2821. Fixed a problem that DistributedShell AM may hang if restarted. Contributed by Varun Vasudev
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d2057047
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d2057047
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d2057047
Branch: refs/heads/YARN-2928
Commit: d2057047a4cb606c1b2d05003cdb82128fd528fc
Parents: 8d096be
Author: Jian He <ji...@apache.org>
Authored: Tue May 19 14:20:31 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 12:18:19 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../pom.xml | 5 +
.../distributedshell/ApplicationMaster.java | 88 ++++++++++---
.../distributedshell/TestDSAppMaster.java | 130 +++++++++++++++++++
4 files changed, 205 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2057047/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e5a0b95..7fbc09a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -501,6 +501,9 @@ Release 2.8.0 - UNRELEASED
YARN-3302. TestDockerContainerExecutor should run automatically if it can
detect docker in the usual place (Ravindra Kumar Naik via raviprak)
+ YARN-2821. Fixed a problem that DistributedShell AM may hang if restarted.
+ (Varun Vasudev via jianhe)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2057047/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
index 18e325c..713f12b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
@@ -126,6 +126,11 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2057047/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index add34af..13bf500 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -30,10 +30,12 @@ import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.Vector;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -177,7 +179,7 @@ public class ApplicationMaster {
public static enum DSEvent {
DS_APP_ATTEMPT_START, DS_APP_ATTEMPT_END, DS_CONTAINER_START, DS_CONTAINER_END
}
-
+
@VisibleForTesting
@Private
public static enum DSEntity {
@@ -220,7 +222,7 @@ public class ApplicationMaster {
private static ExecutorService threadPool =
Executors.newCachedThreadPool(
new ThreadFactoryBuilder().setNameFormat("TimelineService #%d")
- .build());
+ .build());
// App Master configuration
// No. of containers to run shell command on
@@ -290,6 +292,10 @@ public class ApplicationMaster {
private final String linux_bash_command = "bash";
private final String windows_command = "cmd /c";
+ @VisibleForTesting
+ protected final Set<ContainerId> launchedContainers =
+ Collections.newSetFromMap(new ConcurrentHashMap<ContainerId, Boolean>());
+
/**
* @param args Command line args
*/
@@ -304,7 +310,7 @@ public class ApplicationMaster {
}
appMaster.run();
result = appMaster.finish();
-
+
shutdownAndAwaitTermination();
} catch (Throwable t) {
LOG.fatal("Error running ApplicationMaster", t);
@@ -319,7 +325,7 @@ public class ApplicationMaster {
System.exit(2);
}
}
-
+
//TODO remove threadPool after adding non-blocking call in TimelineClient
private static void shutdownAndAwaitTermination() {
threadPool.shutdown();
@@ -639,7 +645,7 @@ public class ApplicationMaster {
// resource manager
int maxMem = response.getMaximumResourceCapability().getMemory();
LOG.info("Max mem capability of resources in this cluster " + maxMem);
-
+
int maxVCores = response.getMaximumResourceCapability().getVirtualCores();
LOG.info("Max vcores capability of resources in this cluster " + maxVCores);
@@ -662,8 +668,12 @@ public class ApplicationMaster {
response.getContainersFromPreviousAttempts();
LOG.info(appAttemptID + " received " + previousAMRunningContainers.size()
+ " previous attempts' running containers on AM registration.");
+ for(Container container: previousAMRunningContainers) {
+ launchedContainers.add(container.getId());
+ }
numAllocatedContainers.addAndGet(previousAMRunningContainers.size());
+
int numTotalContainersToRequest =
numTotalContainers - previousAMRunningContainers.size();
// Setup ask for containers from RM
@@ -758,7 +768,7 @@ public class ApplicationMaster {
FinalApplicationStatus appStatus;
String appMessage = null;
boolean success = true;
- if (numFailedContainers.get() == 0 &&
+ if (numFailedContainers.get() == 0 &&
numCompletedContainers.get() == numTotalContainers) {
appStatus = FinalApplicationStatus.SUCCEEDED;
} else {
@@ -777,7 +787,7 @@ public class ApplicationMaster {
} catch (IOException e) {
LOG.error("Failed to unregister application", e);
}
-
+
amRMClient.stop();
// Stop Timeline Client
@@ -787,8 +797,9 @@ public class ApplicationMaster {
return success;
}
-
- private class RMCallbackHandler implements AMRMClientAsync.CallbackHandler {
+
+ @VisibleForTesting
+ class RMCallbackHandler implements AMRMClientAsync.CallbackHandler {
@SuppressWarnings("unchecked")
@Override
public void onContainersCompleted(List<ContainerStatus> completedContainers) {
@@ -803,6 +814,14 @@ public class ApplicationMaster {
// non complete containers should not be here
assert (containerStatus.getState() == ContainerState.COMPLETE);
+ // ignore containers we know nothing about - probably from a previous
+ // attempt
+ if (!launchedContainers.contains(containerStatus.getContainerId())) {
+ LOG.info("Ignoring completed status of "
+ + containerStatus.getContainerId()
+ + "; unknown container(probably launched by previous attempt)");
+ continue;
+ }
// increment counters for completed/failed containers
int exitStatus = containerStatus.getExitStatus();
@@ -838,7 +857,7 @@ public class ApplicationMaster {
}
}
}
-
+
// ask for more containers if any failed
int askCount = numTotalContainers - numRequestedContainers.get();
numRequestedContainers.addAndGet(askCount);
@@ -849,7 +868,7 @@ public class ApplicationMaster {
amRMClient.addContainerRequest(containerAsk);
}
}
-
+
if (numCompletedContainers.get() == numTotalContainers) {
done = true;
}
@@ -873,14 +892,13 @@ public class ApplicationMaster {
// + ", containerToken"
// +allocatedContainer.getContainerToken().getIdentifier().toString());
- LaunchContainerRunnable runnableLaunchContainer =
- new LaunchContainerRunnable(allocatedContainer, containerListener);
- Thread launchThread = new Thread(runnableLaunchContainer);
+ Thread launchThread = createLaunchContainerThread(allocatedContainer);
// launch and start the container on a separate thread to keep
// the main thread unblocked
// as all containers may not be allocated at one go.
launchThreads.add(launchThread);
+ launchedContainers.add(allocatedContainer.getId());
launchThread.start();
}
}
@@ -1160,7 +1178,7 @@ public class ApplicationMaster {
org.apache.commons.io.IOUtils.closeQuietly(ds);
}
}
-
+
private static void publishContainerStartEvent(
final TimelineClient timelineClient, Container container, String domainId,
UserGroupInformation ugi) {
@@ -1249,7 +1267,8 @@ public class ApplicationMaster {
private static void publishContainerStartEventOnNewTimelineServiceBase(
final TimelineClient timelineClient, Container container, String domainId,
UserGroupInformation ugi) {
- final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
+ entity =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
entity.setId(container.getId().toString());
entity.setType(DSEntity.DS_CONTAINER.toString());
@@ -1277,8 +1296,8 @@ public class ApplicationMaster {
final String domainId, final UserGroupInformation ugi) {
Runnable publishWrapper = new Runnable() {
public void run() {
- publishContainerEndEventOnNewTimelineServiceBase(timelineClient,
- container, domainId, ugi);
+ publishContainerEndEventOnNewTimelineServiceBase(timelineClient,
+ container, domainId, ugi);
}
};
threadPool.execute(publishWrapper);
@@ -1287,14 +1306,15 @@ public class ApplicationMaster {
private static void publishContainerEndEventOnNewTimelineServiceBase(
final TimelineClient timelineClient, final ContainerStatus container,
final String domainId, final UserGroupInformation ugi) {
- final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
+ entity =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
entity.setId(container.getContainerId().toString());
entity.setType(DSEntity.DS_CONTAINER.toString());
//entity.setDomainId(domainId);
entity.addInfo("user", ugi.getShortUserName());
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event =
- new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent();
+ new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent();
event.setTimestamp(System.currentTimeMillis());
event.setId(DSEvent.DS_CONTAINER_END.toString());
event.addInfo("State", container.getState().name());
@@ -1325,7 +1345,8 @@ public class ApplicationMaster {
private static void publishApplicationAttemptEventOnNewTimelineServiceBase(
final TimelineClient timelineClient, String appAttemptId,
DSEvent appEvent, String domainId, UserGroupInformation ugi) {
- final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
+ entity =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
entity.setId(appAttemptId);
entity.setType(DSEntity.DS_APP_ATTEMPT.toString());
@@ -1347,4 +1368,29 @@ public class ApplicationMaster {
}
}
+ RMCallbackHandler getRMCallbackHandler() {
+ return new RMCallbackHandler();
+ }
+
+ @VisibleForTesting
+ void setAmRMClient(AMRMClientAsync client) {
+ this.amRMClient = client;
+ }
+
+ @VisibleForTesting
+ int getNumCompletedContainers() {
+ return numCompletedContainers.get();
+ }
+
+ @VisibleForTesting
+ boolean getDone() {
+ return done;
+ }
+
+ @VisibleForTesting
+ Thread createLaunchContainerThread(Container allocatedContainer) {
+ LaunchContainerRunnable runnableLaunchContainer =
+ new LaunchContainerRunnable(allocatedContainer, containerListener);
+ return new Thread(runnableLaunchContainer);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2057047/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java
index 11e840a..0fed14d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java
@@ -20,13 +20,143 @@ package org.apache.hadoop.yarn.applications.distributedshell;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.Assert;
import org.junit.Test;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A bunch of tests to make sure that the container allocations
+ * and releases occur correctly.
+ */
public class TestDSAppMaster {
+ static class TestAppMaster extends ApplicationMaster {
+ private int threadsLaunched = 0;
+
+ @Override
+ protected Thread createLaunchContainerThread(Container allocatedContainer) {
+ threadsLaunched++;
+ launchedContainers.add(allocatedContainer.getId());
+ return new Thread();
+ }
+
+ void setNumTotalContainers(int numTotalContainers) {
+ this.numTotalContainers = numTotalContainers;
+ }
+
+ int getAllocatedContainers() {
+ return this.numAllocatedContainers.get();
+ }
+
+ @Override
+ void startTimelineClient(final Configuration conf) throws YarnException,
+ IOException, InterruptedException {
+ timelineClient = null;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testDSAppMasterAllocateHandler() throws Exception {
+
+ TestAppMaster master = new TestAppMaster();
+ int targetContainers = 2;
+ AMRMClientAsync mockClient = Mockito.mock(AMRMClientAsync.class);
+ master.setAmRMClient(mockClient);
+ master.setNumTotalContainers(targetContainers);
+ Mockito.doNothing().when(mockClient)
+ .addContainerRequest(Matchers.any(AMRMClient.ContainerRequest.class));
+
+ ApplicationMaster.RMCallbackHandler handler = master.getRMCallbackHandler();
+
+ List<Container> containers = new ArrayList<>(1);
+ ContainerId id1 = BuilderUtils.newContainerId(1, 1, 1, 1);
+ containers.add(generateContainer(id1));
+
+ master.numRequestedContainers.set(targetContainers);
+
+ // first allocate a single container, everything should be fine
+ handler.onContainersAllocated(containers);
+ Assert.assertEquals("Wrong container allocation count", 1,
+ master.getAllocatedContainers());
+ Mockito.verifyZeroInteractions(mockClient);
+ Assert.assertEquals("Incorrect number of threads launched", 1,
+ master.threadsLaunched);
+
+ // now send 3 extra containers
+ containers.clear();
+ ContainerId id2 = BuilderUtils.newContainerId(1, 1, 1, 2);
+ containers.add(generateContainer(id2));
+ ContainerId id3 = BuilderUtils.newContainerId(1, 1, 1, 3);
+ containers.add(generateContainer(id3));
+ ContainerId id4 = BuilderUtils.newContainerId(1, 1, 1, 4);
+ containers.add(generateContainer(id4));
+ handler.onContainersAllocated(containers);
+ Assert.assertEquals("Wrong final container allocation count", 4,
+ master.getAllocatedContainers());
+
+ Assert.assertEquals("Incorrect number of threads launched", 4,
+ master.threadsLaunched);
+
+ // make sure we handle completion events correctly
+ List<ContainerStatus> status = new ArrayList<>();
+ status.add(generateContainerStatus(id1, ContainerExitStatus.SUCCESS));
+ status.add(generateContainerStatus(id2, ContainerExitStatus.SUCCESS));
+ status.add(generateContainerStatus(id3, ContainerExitStatus.ABORTED));
+ status.add(generateContainerStatus(id4, ContainerExitStatus.ABORTED));
+ handler.onContainersCompleted(status);
+
+ Assert.assertEquals("Unexpected number of completed containers",
+ targetContainers, master.getNumCompletedContainers());
+ Assert.assertTrue("Master didn't finish containers as expected",
+ master.getDone());
+
+ // test for events from containers we know nothing about
+ // these events should be ignored
+ status = new ArrayList<>();
+ ContainerId id5 = BuilderUtils.newContainerId(1, 1, 1, 5);
+ status.add(generateContainerStatus(id5, ContainerExitStatus.ABORTED));
+ Assert.assertEquals("Unexpected number of completed containers",
+ targetContainers, master.getNumCompletedContainers());
+ Assert.assertTrue("Master didn't finish containers as expected",
+ master.getDone());
+ status.add(generateContainerStatus(id5, ContainerExitStatus.SUCCESS));
+ Assert.assertEquals("Unexpected number of completed containers",
+ targetContainers, master.getNumCompletedContainers());
+ Assert.assertTrue("Master didn't finish containers as expected",
+ master.getDone());
+ }
+
+ private Container generateContainer(ContainerId cid) {
+ return Container.newInstance(cid, NodeId.newInstance("host", 5000),
+ "host:80", Resource.newInstance(1024, 1), Priority.newInstance(0), null);
+ }
+
+ private ContainerStatus
+ generateContainerStatus(ContainerId id, int exitStatus) {
+ return ContainerStatus.newInstance(id, ContainerState.COMPLETE, "",
+ exitStatus);
+ }
+
@Test
public void testTimelineClientInDSAppMaster() throws Exception {
ApplicationMaster appMaster = new ApplicationMaster();
[27/50] [abbrv] hadoop git commit: HADOOP-11970. Replace uses of
ThreadLocal with JDK7 ThreadLocalRandom (Sean Busbey via Colin P.
McCabe)
Posted by zj...@apache.org.
HADOOP-11970. Replace uses of ThreadLocal<Random> with JDK7 ThreadLocalRandom (Sean Busbey via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0bd4425d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0bd4425d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0bd4425d
Branch: refs/heads/YARN-2928
Commit: 0bd4425d9276ebcf2abbcdad8a9993a736260cd5
Parents: 509e6cb
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue May 19 10:49:17 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:52:09 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../apache/hadoop/io/retry/RetryPolicies.java | 14 ++----
.../java/org/apache/hadoop/hdfs/DFSClient.java | 4 +-
.../org/apache/hadoop/hdfs/DFSInputStream.java | 5 ++-
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 16 +------
.../server/blockmanagement/BlockManager.java | 8 ++--
.../server/blockmanagement/DatanodeManager.java | 3 +-
.../server/blockmanagement/Host2NodesMap.java | 4 +-
.../hdfs/server/datanode/BPServiceActor.java | 10 ++---
.../hdfs/server/datanode/DirectoryScanner.java | 5 ++-
.../datanode/metrics/DataNodeMetrics.java | 5 ++-
.../hadoop/hdfs/server/namenode/NNStorage.java | 3 +-
.../hdfs/server/namenode/NamenodeFsck.java | 4 +-
.../hadoop/hdfs/TestAppendSnapshotTruncate.java | 22 +++++-----
.../apache/hadoop/hdfs/TestRollingUpgrade.java | 6 ++-
.../blockmanagement/TestReplicationPolicy.java | 46 ++++++++++++--------
.../hdfs/server/namenode/TestFileTruncate.java | 10 ++---
.../hdfs/server/namenode/ha/TestDNFencing.java | 4 +-
.../hdfs/server/namenode/ha/TestHAAppend.java | 6 +--
.../hadoop/hdfs/util/TestByteArrayManager.java | 14 +++---
.../sharedcache/SharedCacheUploader.java | 11 +----
21 files changed, 101 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index baf9a0f..10da9d7 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -595,6 +595,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11812. Implement listLocatedStatus for ViewFileSystem to speed up
split calculation (gera)
+ HADOOP-11970. Replace uses of ThreadLocal<Random> with JDK7
+ ThreadLocalRandom. (Sean Busbey via Colin P. McCabe)
+
BUG FIXES
HADOOP-11802: DomainSocketWatcher thread terminates sometimes after there
is an I/O error during requestShortCircuitShm (cmccabe)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
index 14ded8e..a86f443 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
@@ -28,7 +28,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
@@ -47,13 +47,6 @@ public class RetryPolicies {
public static final Log LOG = LogFactory.getLog(RetryPolicies.class);
- private static ThreadLocal<Random> RANDOM = new ThreadLocal<Random>() {
- @Override
- protected Random initialValue() {
- return new Random();
- }
- };
-
/**
* <p>
* Try once, and fail by re-throwing the exception.
@@ -321,7 +314,8 @@ public class RetryPolicies {
}
//calculate sleep time and return.
- final double ratio = RANDOM.get().nextDouble() + 0.5;//0.5 <= ratio <=1.5
+ // ensure 0.5 <= ratio <=1.5
+ final double ratio = ThreadLocalRandom.current().nextDouble() + 0.5;
final long sleepTime = Math.round(p.sleepMillis * ratio);
return new RetryAction(RetryAction.RetryDecision.RETRY, sleepTime);
}
@@ -610,7 +604,7 @@ public class RetryPolicies {
private static long calculateExponentialTime(long time, int retries,
long cap) {
long baseTime = Math.min(time * (1L << retries), cap);
- return (long) (baseTime * (RANDOM.get().nextDouble() + 0.5));
+ return (long) (baseTime * (ThreadLocalRandom.current().nextDouble() + 0.5));
}
private static long calculateExponentialTime(long time, int retries) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index f32702e..a2b9760 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -48,6 +48,7 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -316,7 +317,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
this.authority = nameNodeUri == null? "null": nameNodeUri.getAuthority();
this.clientName = "DFSClient_" + dfsClientConf.getTaskId() + "_" +
- DFSUtil.getRandom().nextInt() + "_" + Thread.currentThread().getId();
+ ThreadLocalRandom.current().nextInt() + "_" +
+ Thread.currentThread().getId();
int numResponseToDrop = conf.getInt(
DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY,
DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 3290223..d1e0b9a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -40,6 +40,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.Future;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -976,7 +977,9 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
// expanded to 9000ms.
final int timeWindow = dfsClient.getConf().getTimeWindow();
double waitTime = timeWindow * failures + // grace period for the last round of attempt
- timeWindow * (failures + 1) * DFSUtil.getRandom().nextDouble(); // expanding time window for each failure
+ // expanding time window for each failure
+ timeWindow * (failures + 1) *
+ ThreadLocalRandom.current().nextDouble();
DFSClient.LOG.warn("DFS chooseDataNode: got # " + (failures + 1) + " IOException, will wait for " + waitTime + " msec.");
Thread.sleep((long)waitTime);
} catch (InterruptedException iex) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
index a925a60..5f501c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
@@ -50,8 +50,8 @@ import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Random;
import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
import javax.net.SocketFactory;
@@ -103,12 +103,6 @@ public class DFSUtil {
public static final Log LOG = LogFactory.getLog(DFSUtil.class.getName());
private DFSUtil() { /* Hidden constructor */ }
- private static final ThreadLocal<Random> RANDOM = new ThreadLocal<Random>() {
- @Override
- protected Random initialValue() {
- return new Random();
- }
- };
private static final ThreadLocal<SecureRandom> SECURE_RANDOM = new ThreadLocal<SecureRandom>() {
@Override
@@ -117,11 +111,6 @@ public class DFSUtil {
}
};
- /** @return a pseudo random number generator. */
- public static Random getRandom() {
- return RANDOM.get();
- }
-
/** @return a pseudo secure random number generator. */
public static SecureRandom getSecureRandom() {
return SECURE_RANDOM.get();
@@ -130,9 +119,8 @@ public class DFSUtil {
/** Shuffle the elements in the given array. */
public static <T> T[] shuffle(final T[] array) {
if (array != null && array.length > 0) {
- final Random random = getRandom();
for (int n = array.length; n > 1; ) {
- final int randomIndex = random.nextInt(n);
+ final int randomIndex = ThreadLocalRandom.current().nextInt(n);
n--;
if (n != randomIndex) {
final T tmp = array[randomIndex];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 6d5808e..8012f71 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -35,6 +35,7 @@ import java.util.Queue;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -1027,7 +1028,8 @@ public class BlockManager {
return new BlocksWithLocations(new BlockWithLocations[0]);
}
Iterator<BlockInfoContiguous> iter = node.getBlockIterator();
- int startBlock = DFSUtil.getRandom().nextInt(numBlocks); // starting from a random block
+ // starting from a random block
+ int startBlock = ThreadLocalRandom.current().nextInt(numBlocks);
// skip blocks
for(int i=0; i<startBlock; i++) {
iter.next();
@@ -1669,7 +1671,7 @@ public class BlockManager {
// switch to a different node randomly
// this to prevent from deterministically selecting the same node even
// if the node failed to replicate the block on previous iterations
- if(DFSUtil.getRandom().nextBoolean())
+ if(ThreadLocalRandom.current().nextBoolean())
srcNode = node;
}
if(numReplicas != null)
@@ -1920,7 +1922,7 @@ public class BlockManager {
datanodeManager.getBlocksPerPostponedMisreplicatedBlocksRescan();
long base = getPostponedMisreplicatedBlocksCount() - blocksPerRescan;
if (base > 0) {
- startIndex = DFSUtil.getRandom().nextLong() % (base+1);
+ startIndex = ThreadLocalRandom.current().nextLong() % (base+1);
if (startIndex < 0) {
startIndex += (base+1);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 7769a78..01f7972 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -51,6 +51,7 @@ import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.*;
+import java.util.concurrent.ThreadLocalRandom;
/**
* Manage datanodes, include decommission and other activities.
@@ -457,7 +458,7 @@ public class DatanodeManager {
// Try something rack local.
if (node == null && !rackNodes.isEmpty()) {
node = (DatanodeDescriptor) (rackNodes
- .get(DFSUtil.getRandom().nextInt(rackNodes.size())));
+ .get(ThreadLocalRandom.current().nextInt(rackNodes.size())));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java
index 420c141..289b79a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/Host2NodesMap.java
@@ -20,12 +20,12 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.DFSUtil;
/** A map from host names to datanode descriptors. */
@InterfaceAudience.Private
@@ -161,7 +161,7 @@ class Host2NodesMap {
return nodes[0];
}
// more than one node
- return nodes[DFSUtil.getRandom().nextInt(nodes.length)];
+ return nodes[ThreadLocalRandom.current().nextInt(nodes.length)];
} finally {
hostmapLock.readLock().unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index 5bc505f..63a0bb6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -28,13 +28,13 @@ import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
import com.google.common.base.Joiner;
import org.apache.commons.logging.Log;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.hdfs.client.BlockReportOptions;
-import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -122,7 +122,7 @@ class BPServiceActor implements Runnable {
this.dn = bpos.getDataNode();
this.nnAddr = nnAddr;
this.dnConf = dn.getDnConf();
- prevBlockReportId = DFSUtil.getRandom().nextLong();
+ prevBlockReportId = ThreadLocalRandom.current().nextLong();
scheduler = new Scheduler(dnConf.heartBeatInterval, dnConf.blockReportInterval);
}
@@ -409,7 +409,7 @@ class BPServiceActor implements Runnable {
// not send a 0 value ourselves.
prevBlockReportId++;
while (prevBlockReportId == 0) {
- prevBlockReportId = DFSUtil.getRandom().nextLong();
+ prevBlockReportId = ThreadLocalRandom.current().nextLong();
}
return prevBlockReportId;
}
@@ -1054,7 +1054,7 @@ class BPServiceActor implements Runnable {
if (delay > 0) { // send BR after random delay
// Numerical overflow is possible here and is okay.
nextBlockReportTime =
- monotonicNow() + DFSUtil.getRandom().nextInt((int) (delay));
+ monotonicNow() + ThreadLocalRandom.current().nextInt((int) (delay));
} else { // send at next heartbeat
nextBlockReportTime = monotonicNow();
}
@@ -1073,7 +1073,7 @@ class BPServiceActor implements Runnable {
// time before we start the periodic block reports.
if (resetBlockReportTime) {
nextBlockReportTime = monotonicNow() +
- DFSUtil.getRandom().nextInt((int)(blockReportIntervalMs));
+ ThreadLocalRandom.current().nextInt((int)(blockReportIntervalMs));
resetBlockReportTime = false;
} else {
/* say the last block report was at 8:20:14. The current report
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index 8453094..3383d0e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -31,6 +31,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -41,7 +42,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
@@ -327,7 +327,8 @@ public class DirectoryScanner implements Runnable {
void start() {
shouldRun = true;
- long offset = DFSUtil.getRandom().nextInt((int) (scanPeriodMsecs/1000L)) * 1000L; //msec
+ long offset = ThreadLocalRandom.current().nextInt(
+ (int) (scanPeriodMsecs/1000L)) * 1000L; //msec
long firstScanTime = Time.now() + offset;
LOG.info("Periodic Directory Tree Verification scan starting at "
+ firstScanTime + " with interval " + scanPeriodMsecs);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
index 2e62b3c..a0f25da 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
@@ -22,7 +22,6 @@ import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.annotation.Metric;
import org.apache.hadoop.metrics2.annotation.Metrics;
@@ -33,6 +32,8 @@ import org.apache.hadoop.metrics2.lib.MutableQuantiles;
import org.apache.hadoop.metrics2.lib.MutableRate;
import org.apache.hadoop.metrics2.source.JvmMetrics;
+import java.util.concurrent.ThreadLocalRandom;
+
/**
*
* This class is for maintaining the various DataNode statistics
@@ -177,7 +178,7 @@ public class DataNodeMetrics {
MetricsSystem ms = DefaultMetricsSystem.instance();
JvmMetrics jm = JvmMetrics.create("DataNode", sessionId, ms);
String name = "DataNodeActivity-"+ (dnName.isEmpty()
- ? "UndefinedDataNodeName"+ DFSUtil.getRandom().nextInt()
+ ? "UndefinedDataNodeName"+ ThreadLocalRandom.current().nextInt()
: dnName.replace(':', '-'));
// Percentile measurement is off by default, by watching no intervals
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
index 26a13bd..14647f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
@@ -32,6 +32,7 @@ import java.util.List;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ThreadLocalRandom;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
@@ -599,7 +600,7 @@ public class NNStorage extends Storage implements Closeable,
private static int newNamespaceID() {
int newID = 0;
while(newID == 0)
- newID = DFSUtil.getRandom().nextInt(0x7FFFFFFF); // use 31 bits only
+ newID = ThreadLocalRandom.current().nextInt(0x7FFFFFFF); // use 31 bits
return newID;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 44dba28..0daf367 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -32,6 +32,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;
+import java.util.concurrent.ThreadLocalRandom;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
@@ -44,7 +45,6 @@ import org.apache.hadoop.hdfs.BlockReader;
import org.apache.hadoop.hdfs.BlockReaderFactory;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.RemotePeerFactory;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.net.Peer;
@@ -933,7 +933,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
}
DatanodeInfo chosenNode;
do {
- chosenNode = nodes[DFSUtil.getRandom().nextInt(nodes.length)];
+ chosenNode = nodes[ThreadLocalRandom.current().nextInt(nodes.length)];
} while (deadNodes.contains(chosenNode));
return chosenNode;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendSnapshotTruncate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendSnapshotTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendSnapshotTruncate.java
index e80e14f..d67ceb0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendSnapshotTruncate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendSnapshotTruncate.java
@@ -29,6 +29,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
@@ -214,8 +215,7 @@ public class TestAppendSnapshotTruncate {
@Override
public String call() throws Exception {
- final Random r = DFSUtil.getRandom();
- final int op = r.nextInt(6);
+ final int op = ThreadLocalRandom.current().nextInt(6);
if (op <= 1) {
pauseAllFiles();
try {
@@ -229,7 +229,8 @@ public class TestAppendSnapshotTruncate {
if (keys.length == 0) {
return "NO-OP";
}
- final String snapshot = keys[r.nextInt(keys.length)];
+ final String snapshot = keys[ThreadLocalRandom.current()
+ .nextInt(keys.length)];
final String s = checkSnapshot(snapshot);
if (op == 2) {
@@ -292,13 +293,13 @@ public class TestAppendSnapshotTruncate {
@Override
public String call() throws IOException {
- final Random r = DFSUtil.getRandom();
- final int op = r.nextInt(9);
+ final int op = ThreadLocalRandom.current().nextInt(9);
if (op == 0) {
return checkFullFile();
} else {
- final int nBlocks = r.nextInt(4) + 1;
- final int lastBlockSize = r.nextInt(BLOCK_SIZE) + 1;
+ final int nBlocks = ThreadLocalRandom.current().nextInt(4) + 1;
+ final int lastBlockSize = ThreadLocalRandom.current()
+ .nextInt(BLOCK_SIZE) + 1;
final int nBytes = nBlocks*BLOCK_SIZE + lastBlockSize;
if (op <= 4) {
@@ -316,8 +317,8 @@ public class TestAppendSnapshotTruncate {
.append(n).append(" bytes to ").append(file.getName());
final byte[] bytes = new byte[n];
- DFSUtil.getRandom().nextBytes(bytes);
-
+ ThreadLocalRandom.current().nextBytes(bytes);
+
{ // write to local file
final FileOutputStream out = new FileOutputStream(localFile, true);
out.write(bytes, 0, bytes.length);
@@ -446,7 +447,6 @@ public class TestAppendSnapshotTruncate {
final Thread t = new Thread(null, new Runnable() {
@Override
public void run() {
- final Random r = DFSUtil.getRandom();
for(State s; !(s = checkErrorState()).isTerminated;) {
if (s == State.RUNNING) {
isCalling.set(true);
@@ -458,7 +458,7 @@ public class TestAppendSnapshotTruncate {
}
isCalling.set(false);
}
- sleep(r.nextInt(100) + 50);
+ sleep(ThreadLocalRandom.current().nextInt(100) + 50);
}
}
}, name);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
index 8baebd8..c4c890f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs;
import java.io.File;
import java.io.IOException;
+import java.util.concurrent.ThreadLocalRandom;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -272,7 +273,7 @@ public class TestRollingUpgrade {
final Path file = new Path(foo, "file");
final byte[] data = new byte[1024];
- DFSUtil.getRandom().nextBytes(data);
+ ThreadLocalRandom.current().nextBytes(data);
final FSDataOutputStream out = cluster.getFileSystem().create(file);
out.write(data, 0, data.length);
out.close();
@@ -320,7 +321,8 @@ public class TestRollingUpgrade {
Assert.assertTrue(dfs.exists(bar));
//truncate a file
- final int newLength = DFSUtil.getRandom().nextInt(data.length - 1) + 1;
+ final int newLength = ThreadLocalRandom.current().nextInt(data.length - 1)
+ + 1;
dfs.truncate(file, newLength);
TestFileTruncate.checkBlockRecovery(file, dfs);
AppendTestUtil.checkFullFile(dfs, file, newLength, data);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index f117ef7..3226578 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -36,6 +36,7 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
@@ -75,7 +76,6 @@ public class TestReplicationPolicy {
((Log4JLogger)BlockPlacementPolicy.LOG).getLogger().setLevel(Level.ALL);
}
- private final Random random = DFSUtil.getRandom();
private static final int BLOCK_SIZE = 1024;
private static final int NUM_OF_DATANODES = 6;
private static NetworkTopology cluster;
@@ -850,15 +850,17 @@ public class TestReplicationPolicy {
.getNamesystem().getBlockManager().neededReplications;
for (int i = 0; i < 100; i++) {
// Adding the blocks directly to normal priority
- neededReplications.add(new Block(random.nextLong()), 2, 0, 3);
+ neededReplications.add(new Block(ThreadLocalRandom.current()
+ .nextLong()), 2, 0, 3);
}
// Lets wait for the replication interval, to start process normal
// priority blocks
Thread.sleep(DFS_NAMENODE_REPLICATION_INTERVAL);
// Adding the block directly to high priority list
- neededReplications.add(new Block(random.nextLong()), 1, 0, 3);
-
+ neededReplications.add(new Block(ThreadLocalRandom.current().nextLong()),
+ 1, 0, 3);
+
// Lets wait for the replication interval
Thread.sleep(DFS_NAMENODE_REPLICATION_INTERVAL);
@@ -880,19 +882,24 @@ public class TestReplicationPolicy {
for (int i = 0; i < 5; i++) {
// Adding QUEUE_HIGHEST_PRIORITY block
- underReplicatedBlocks.add(new Block(random.nextLong()), 1, 0, 3);
+ underReplicatedBlocks.add(new Block(ThreadLocalRandom.current()
+ .nextLong()), 1, 0, 3);
// Adding QUEUE_VERY_UNDER_REPLICATED block
- underReplicatedBlocks.add(new Block(random.nextLong()), 2, 0, 7);
+ underReplicatedBlocks.add(new Block(ThreadLocalRandom.current()
+ .nextLong()), 2, 0, 7);
// Adding QUEUE_REPLICAS_BADLY_DISTRIBUTED block
- underReplicatedBlocks.add(new Block(random.nextLong()), 6, 0, 6);
+ underReplicatedBlocks.add(new Block(ThreadLocalRandom.current()
+ .nextLong()), 6, 0, 6);
// Adding QUEUE_UNDER_REPLICATED block
- underReplicatedBlocks.add(new Block(random.nextLong()), 5, 0, 6);
+ underReplicatedBlocks.add(new Block(ThreadLocalRandom.current()
+ .nextLong()), 5, 0, 6);
// Adding QUEUE_WITH_CORRUPT_BLOCKS block
- underReplicatedBlocks.add(new Block(random.nextLong()), 0, 0, 3);
+ underReplicatedBlocks.add(new Block(ThreadLocalRandom.current()
+ .nextLong()), 0, 0, 3);
}
// Choose 6 blocks from UnderReplicatedBlocks. Then it should pick 5 blocks
@@ -908,7 +915,8 @@ public class TestReplicationPolicy {
assertTheChosenBlocks(chosenBlocks, 0, 4, 5, 1, 0);
// Adding QUEUE_HIGHEST_PRIORITY
- underReplicatedBlocks.add(new Block(random.nextLong()), 1, 0, 3);
+ underReplicatedBlocks.add(new Block(ThreadLocalRandom.current().nextLong()),
+ 1, 0, 3);
// Choose 10 blocks from UnderReplicatedBlocks. Then it should pick 1 block from
// QUEUE_HIGHEST_PRIORITY, 4 blocks from QUEUE_REPLICAS_BADLY_DISTRIBUTED
@@ -1100,9 +1108,9 @@ public class TestReplicationPolicy {
public void testUpdateDoesNotCauseSkippedReplication() {
UnderReplicatedBlocks underReplicatedBlocks = new UnderReplicatedBlocks();
- Block block1 = new Block(random.nextLong());
- Block block2 = new Block(random.nextLong());
- Block block3 = new Block(random.nextLong());
+ Block block1 = new Block(ThreadLocalRandom.current().nextLong());
+ Block block2 = new Block(ThreadLocalRandom.current().nextLong());
+ Block block3 = new Block(ThreadLocalRandom.current().nextLong());
// Adding QUEUE_VERY_UNDER_REPLICATED block
final int block1CurReplicas = 2;
@@ -1149,8 +1157,8 @@ public class TestReplicationPolicy {
BlockManager bm = new BlockManager(mockNS, new HdfsConfiguration());
UnderReplicatedBlocks underReplicatedBlocks = bm.neededReplications;
- Block block1 = new Block(random.nextLong());
- Block block2 = new Block(random.nextLong());
+ Block block1 = new Block(ThreadLocalRandom.current().nextLong());
+ Block block2 = new Block(ThreadLocalRandom.current().nextLong());
// Adding QUEUE_UNDER_REPLICATED block
underReplicatedBlocks.add(block1, 0, 1, 1);
@@ -1195,8 +1203,8 @@ public class TestReplicationPolicy {
BlockManager bm = new BlockManager(mockNS, new HdfsConfiguration());
UnderReplicatedBlocks underReplicatedBlocks = bm.neededReplications;
- Block block1 = new Block(random.nextLong());
- Block block2 = new Block(random.nextLong());
+ Block block1 = new Block(ThreadLocalRandom.current().nextLong());
+ Block block2 = new Block(ThreadLocalRandom.current().nextLong());
// Adding QUEUE_UNDER_REPLICATED block
underReplicatedBlocks.add(block1, 0, 1, 1);
@@ -1258,8 +1266,8 @@ public class TestReplicationPolicy {
BlockManager bm = new BlockManager(mockNS, new HdfsConfiguration());
UnderReplicatedBlocks underReplicatedBlocks = bm.neededReplications;
- Block block1 = new Block(random.nextLong());
- Block block2 = new Block(random.nextLong());
+ Block block1 = new Block(ThreadLocalRandom.current().nextLong());
+ Block block2 = new Block(ThreadLocalRandom.current().nextLong());
// Adding QUEUE_UNDER_REPLICATED block
underReplicatedBlocks.add(block1, 0, 1, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
index 1f65f78..222f22b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
@@ -31,6 +31,7 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.util.concurrent.ThreadLocalRandom;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -46,7 +47,6 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.AppendTestUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
-import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -164,11 +164,11 @@ public class TestFileTruncate {
fs.mkdirs(dir);
final Path p = new Path(dir, "file");
final byte[] data = new byte[100 * BLOCK_SIZE];
- DFSUtil.getRandom().nextBytes(data);
+ ThreadLocalRandom.current().nextBytes(data);
writeContents(data, data.length, p);
for(int n = data.length; n > 0; ) {
- final int newLength = DFSUtil.getRandom().nextInt(n);
+ final int newLength = ThreadLocalRandom.current().nextInt(n);
final boolean isReady = fs.truncate(p, newLength);
LOG.info("newLength=" + newLength + ", isReady=" + isReady);
assertEquals("File must be closed for truncating at the block boundary",
@@ -193,7 +193,7 @@ public class TestFileTruncate {
fs.allowSnapshot(dir);
final Path p = new Path(dir, "file");
final byte[] data = new byte[BLOCK_SIZE];
- DFSUtil.getRandom().nextBytes(data);
+ ThreadLocalRandom.current().nextBytes(data);
writeContents(data, data.length, p);
final String snapshot = "s0";
fs.createSnapshot(dir, snapshot);
@@ -226,7 +226,7 @@ public class TestFileTruncate {
final Path p = new Path(dir, "file");
final byte[] data = new byte[2 * BLOCK_SIZE];
- DFSUtil.getRandom().nextBytes(data);
+ ThreadLocalRandom.current().nextBytes(data);
writeContents(data, data.length, p);
final int newLength = data.length - 1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
index 42bf46f..04b7b94 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java
@@ -24,6 +24,7 @@ import java.io.PrintWriter;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
import com.google.common.base.Supplier;
import com.google.common.collect.Lists;
@@ -37,7 +38,6 @@ import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.AppendTestUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
-import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.protocol.Block;
@@ -640,7 +640,7 @@ public class TestDNFencing {
Collection<DatanodeStorageInfo> chooseFrom = !first.isEmpty() ? first : second;
List<DatanodeStorageInfo> l = Lists.newArrayList(chooseFrom);
- return l.get(DFSUtil.getRandom().nextInt(l.size()));
+ return l.get(ThreadLocalRandom.current().nextInt(l.size()));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAAppend.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAAppend.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAAppend.java
index e0aad02..7b7100b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAAppend.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAAppend.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode.ha;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
+import java.util.concurrent.ThreadLocalRandom;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -27,7 +28,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.AppendTestUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.server.namenode.TestFileTruncate;
@@ -73,7 +73,7 @@ public class TestHAAppend {
Path fileToTruncate = new Path("/FileToTruncate");
final byte[] data = new byte[1 << 16];
- DFSUtil.getRandom().nextBytes(data);
+ ThreadLocalRandom.current().nextBytes(data);
final int[] appendPos = AppendTestUtil.randomFilePartition(
data.length, COUNT);
final int[] truncatePos = AppendTestUtil.randomFilePartition(
@@ -136,4 +136,4 @@ public class TestHAAppend {
}
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestByteArrayManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestByteArrayManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestByteArrayManager.java
index 1348033..972e51e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestByteArrayManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestByteArrayManager.java
@@ -27,6 +27,7 @@ import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
@@ -34,7 +35,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
-import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.util.ByteArrayManager.Counter;
import org.apache.hadoop.hdfs.util.ByteArrayManager.CounterMap;
@@ -72,7 +72,7 @@ public class TestByteArrayManager {
final long countResetTimePeriodMs = 200L;
final Counter c = new Counter(countResetTimePeriodMs);
- final int n = DFSUtil.getRandom().nextInt(512) + 512;
+ final int n = ThreadLocalRandom.current().nextInt(512) + 512;
final List<Future<Integer>> futures = new ArrayList<Future<Integer>>(n);
final ExecutorService pool = Executors.newFixedThreadPool(32);
@@ -334,7 +334,7 @@ public class TestByteArrayManager {
public void run() {
LOG.info("randomRecycler start");
for(int i = 0; shouldRun(); i++) {
- final int j = DFSUtil.getRandom().nextInt(runners.length);
+ final int j = ThreadLocalRandom.current().nextInt(runners.length);
try {
runners[j].recycle();
} catch (Exception e) {
@@ -440,7 +440,7 @@ public class TestByteArrayManager {
public byte[] call() throws Exception {
final int lower = maxArrayLength == ByteArrayManager.MIN_ARRAY_LENGTH?
0: maxArrayLength >> 1;
- final int arrayLength = DFSUtil.getRandom().nextInt(
+ final int arrayLength = ThreadLocalRandom.current().nextInt(
maxArrayLength - lower) + lower + 1;
final byte[] array = bam.newByteArray(arrayLength);
try {
@@ -496,7 +496,8 @@ public class TestByteArrayManager {
@Override
public void run() {
for(int i = 0; i < n; i++) {
- final boolean isAllocate = DFSUtil.getRandom().nextInt(NUM_RUNNERS) < p;
+ final boolean isAllocate = ThreadLocalRandom.current()
+ .nextInt(NUM_RUNNERS) < p;
if (isAllocate) {
submitAllocate();
} else {
@@ -573,7 +574,6 @@ public class TestByteArrayManager {
+ ", nAllocations=" + nAllocations
+ ", maxArrays=" + maxArrays);
- final Random ran = DFSUtil.getRandom();
final ByteArrayManager[] impls = {
new ByteArrayManager.NewByteArrayWithoutLimit(),
new NewByteArrayWithLimit(maxArrays),
@@ -590,7 +590,7 @@ public class TestByteArrayManager {
for(int j = 0; j < nTrials; j++) {
final int[] sleepTime = new int[nAllocations];
for(int k = 0; k < sleepTime.length; k++) {
- sleepTime[k] = ran.nextInt(100);
+ sleepTime[k] = ThreadLocalRandom.current().nextInt(100);
}
final long elapsed = performanceTest(arrayLength, maxArrays, nThreads,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bd4425d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploader.java
index 050d531..682b272 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploader.java
@@ -22,8 +22,8 @@ import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.UndeclaredThrowableException;
import java.net.URISyntaxException;
-import java.util.Random;
import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -61,13 +61,6 @@ class SharedCacheUploader implements Callable<Boolean> {
new FsPermission((short)00555);
private static final Log LOG = LogFactory.getLog(SharedCacheUploader.class);
- private static final ThreadLocal<Random> randomTl =
- new ThreadLocal<Random>() {
- @Override
- protected Random initialValue() {
- return new Random(System.nanoTime());
- }
- };
private final LocalResource resource;
private final Path localPath;
@@ -267,7 +260,7 @@ class SharedCacheUploader implements Callable<Boolean> {
}
private String getTemporaryFileName(Path path) {
- return path.getName() + "-" + randomTl.get().nextLong();
+ return path.getName() + "-" + ThreadLocalRandom.current().nextLong();
}
@VisibleForTesting
[34/50] [abbrv] hadoop git commit: HADOOP-12000. cannot use
--java-home in test-patch (aw)
Posted by zj...@apache.org.
HADOOP-12000. cannot use --java-home in test-patch (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/48e840f7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/48e840f7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/48e840f7
Branch: refs/heads/YARN-2928
Commit: 48e840f7600392838021f0214ac32b10a2852f7d
Parents: d205704
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue May 19 15:24:23 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 12:18:36 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.sh | 2 +-
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48e840f7/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 57fd657..d5d6c26 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -679,7 +679,7 @@ function parse_args
--issue-re=*)
ISSUE_RE=${i#*=}
;;
- --java-home)
+ --java-home=*)
JAVA_HOME=${i#*=}
;;
--jenkins)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48e840f7/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 4621f80..3e7cb39 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -719,6 +719,8 @@ Release 2.8.0 - UNRELEASED
HADOOP-11963. Metrics documentation for FSNamesystem misspells
PendingDataNodeMessageCount. (Anu Engineer via cnauroth)
+ HADOOP-12000. cannot use --java-home in test-patch (aw)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
[46/50] [abbrv] hadoop git commit: YARN-3694. Fix dead link for
TimelineServer REST API. Contributed by Jagadesh Kiran N.
Posted by zj...@apache.org.
YARN-3694. Fix dead link for TimelineServer REST API. Contributed by Jagadesh Kiran N.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/72ff4607
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/72ff4607
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/72ff4607
Branch: refs/heads/YARN-2928
Commit: 72ff46079d87796fed4a6b7ce3cd1b4567674de4
Parents: 5f2297c
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu May 21 23:14:44 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 12:42:58 2015 -0700
----------------------------------------------------------------------
hadoop-project/src/site/site.xml | 2 +-
hadoop-yarn-project/CHANGES.txt | 3 +++
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72ff4607/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 7234881..f3bb458 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -134,7 +134,7 @@
<item name="Introduction" href="hadoop-yarn/hadoop-yarn-site/WebServicesIntro.html"/>
<item name="Resource Manager" href="hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html"/>
<item name="Node Manager" href="hadoop-yarn/hadoop-yarn-site/NodeManagerRest.html"/>
- <item name="Timeline Server" href="TimelineServer.html#Timeline_Server_REST_API_v1"/>
+ <item name="Timeline Server" href="hadoop-yarn/hadoop-yarn-site/TimelineServer.html#Timeline_Server_REST_API_v1"/>
</menu>
<menu name="Hadoop Compatible File Systems" inherit="top">
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72ff4607/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 863211c..cb3ecb7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -616,6 +616,9 @@ Release 2.7.1 - UNRELEASED
YARN-3609. Load node labels from storage inside RM serviceStart. (Wangda
Tan via jianhe)
+ YARN-3694. Fix dead link for TimelineServer REST API.
+ (Jagadesh Kiran N via aajisaka)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
[44/50] [abbrv] hadoop git commit: HADOOP-11772. RPC Invoker relies
on static ClientCache which has synchronized(this) blocks. Contributed by
Haohui Mai.
Posted by zj...@apache.org.
HADOOP-11772. RPC Invoker relies on static ClientCache which has synchronized(this) blocks. Contributed by Haohui Mai.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/af9c5595
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/af9c5595
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/af9c5595
Branch: refs/heads/YARN-2928
Commit: af9c559598ed3e796310f5283774d06e771bf4e3
Parents: 458de34
Author: Haohui Mai <wh...@apache.org>
Authored: Wed May 20 20:10:50 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 12:42:57 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../main/java/org/apache/hadoop/ipc/Client.java | 106 ++++++-------------
2 files changed, 35 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af9c5595/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 1624ce2..416b819 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -604,6 +604,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11970. Replace uses of ThreadLocal<Random> with JDK7
ThreadLocalRandom. (Sean Busbey via Colin P. McCabe)
+ HADOOP-11772. RPC Invoker relies on static ClientCache which has
+ synchronized(this) blocks. (wheat9)
+
BUG FIXES
HADOOP-11802: DomainSocketWatcher thread terminates sometimes after there
is an I/O error during requestShortCircuitShm (cmccabe)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af9c5595/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index f28d8a2..feb811e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -43,6 +43,7 @@ import java.util.Iterator;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -56,6 +57,8 @@ import java.util.concurrent.atomic.AtomicLong;
import javax.net.SocketFactory;
import javax.security.sasl.Sasl;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -124,8 +127,8 @@ public class Client {
retryCount.set(rc);
}
- private Hashtable<ConnectionId, Connection> connections =
- new Hashtable<ConnectionId, Connection>();
+ private final Cache<ConnectionId, Connection> connections =
+ CacheBuilder.newBuilder().build();
private Class<? extends Writable> valueClass; // class of call values
private AtomicBoolean running = new AtomicBoolean(true); // if client runs
@@ -1167,13 +1170,7 @@ public class Client {
return;
}
- // release the resources
- // first thing to do;take the connection out of the connection list
- synchronized (connections) {
- if (connections.get(remoteId) == this) {
- connections.remove(remoteId);
- }
- }
+ connections.invalidate(remoteId);
// close the streams and therefore the socket
IOUtils.closeStream(out);
@@ -1260,14 +1257,12 @@ public class Client {
}
// wake up all connections
- synchronized (connections) {
- for (Connection conn : connections.values()) {
- conn.interrupt();
- }
+ for (Connection conn : connections.asMap().values()) {
+ conn.interrupt();
}
// wait until all connections are closed
- while (!connections.isEmpty()) {
+ while (connections.size() > 0) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
@@ -1283,56 +1278,12 @@ public class Client {
*/
public Writable call(Writable param, InetSocketAddress address)
throws IOException {
- return call(RPC.RpcKind.RPC_BUILTIN, param, address);
-
- }
- /** Make a call, passing <code>param</code>, to the IPC server running at
- * <code>address</code>, returning the value. Throws exceptions if there are
- * network problems or if the remote code threw an exception.
- * @deprecated Use {@link #call(RPC.RpcKind, Writable,
- * ConnectionId)} instead
- */
- @Deprecated
- public Writable call(RPC.RpcKind rpcKind, Writable param, InetSocketAddress address)
- throws IOException {
- return call(rpcKind, param, address, null);
- }
-
- /** Make a call, passing <code>param</code>, to the IPC server running at
- * <code>address</code> with the <code>ticket</code> credentials, returning
- * the value.
- * Throws exceptions if there are network problems or if the remote code
- * threw an exception.
- * @deprecated Use {@link #call(RPC.RpcKind, Writable,
- * ConnectionId)} instead
- */
- @Deprecated
- public Writable call(RPC.RpcKind rpcKind, Writable param, InetSocketAddress addr,
- UserGroupInformation ticket) throws IOException {
- ConnectionId remoteId = ConnectionId.getConnectionId(addr, null, ticket, 0,
+ ConnectionId remoteId = ConnectionId.getConnectionId(address, null, null, 0,
conf);
- return call(rpcKind, param, remoteId);
- }
-
- /** Make a call, passing <code>param</code>, to the IPC server running at
- * <code>address</code> which is servicing the <code>protocol</code> protocol,
- * with the <code>ticket</code> credentials and <code>rpcTimeout</code> as
- * timeout, returning the value.
- * Throws exceptions if there are network problems or if the remote code
- * threw an exception.
- * @deprecated Use {@link #call(RPC.RpcKind, Writable,
- * ConnectionId)} instead
- */
- @Deprecated
- public Writable call(RPC.RpcKind rpcKind, Writable param, InetSocketAddress addr,
- Class<?> protocol, UserGroupInformation ticket,
- int rpcTimeout) throws IOException {
- ConnectionId remoteId = ConnectionId.getConnectionId(addr, protocol,
- ticket, rpcTimeout, conf);
- return call(rpcKind, param, remoteId);
+ return call(RpcKind.RPC_BUILTIN, param, remoteId);
+
}
-
/**
* Same as {@link #call(RPC.RpcKind, Writable, InetSocketAddress,
* Class, UserGroupInformation, int, Configuration)}
@@ -1506,15 +1457,14 @@ public class Client {
@InterfaceAudience.Private
@InterfaceStability.Unstable
Set<ConnectionId> getConnectionIds() {
- synchronized (connections) {
- return connections.keySet();
- }
+ return connections.asMap().keySet();
}
/** Get a connection from the pool, or create a new one and add it to the
* pool. Connections to a given ConnectionId are reused. */
- private Connection getConnection(ConnectionId remoteId,
- Call call, int serviceClass, AtomicBoolean fallbackToSimpleAuth)
+ private Connection getConnection(
+ final ConnectionId remoteId,
+ Call call, final int serviceClass, AtomicBoolean fallbackToSimpleAuth)
throws IOException {
if (!running.get()) {
// the client is stopped
@@ -1525,15 +1475,23 @@ public class Client {
* connectionsId object and with set() method. We need to manage the
* refs for keys in HashMap properly. For now its ok.
*/
- do {
- synchronized (connections) {
- connection = connections.get(remoteId);
- if (connection == null) {
- connection = new Connection(remoteId, serviceClass);
- connections.put(remoteId, connection);
- }
+ while(true) {
+ try {
+ connection = connections.get(remoteId, new Callable<Connection>() {
+ @Override
+ public Connection call() throws Exception {
+ return new Connection(remoteId, serviceClass);
+ }
+ });
+ } catch (ExecutionException e) {
+ throw new IOException(e);
+ }
+ if (connection.addCall(call)) {
+ break;
+ } else {
+ connections.invalidate(remoteId);
}
- } while (!connection.addCall(call));
+ }
//we don't invoke the method below inside "synchronized (connections)"
//block above. The reason for that is if the server happens to be slow,
[32/50] [abbrv] hadoop git commit: HDFS-8404. Pending block
replication can get stuck using older genstamp. Contributed by Nathan
Roberts.
Posted by zj...@apache.org.
HDFS-8404. Pending block replication can get stuck using older genstamp. Contributed by Nathan Roberts.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f2ba726
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f2ba726
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f2ba726
Branch: refs/heads/YARN-2928
Commit: 6f2ba726cea17feae90c3d3d808668ddd748bba5
Parents: 0bd4425
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue May 19 13:05:15 2015 -0500
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:52:10 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../server/blockmanagement/BlockManager.java | 17 ++--
.../blockmanagement/TestPendingReplication.java | 98 +++++++++++++++++++-
3 files changed, 108 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f2ba726/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 76888a9..5bcaddd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -878,6 +878,9 @@ Release 2.7.1 - UNRELEASED
HDFS-8405. Fix a typo in NamenodeFsck. (Takanobu Asanuma via szetszwo)
+ HDFS-8404. Pending block replication can get stuck using older genstamp
+ (Nathan Roberts via kihwal)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f2ba726/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 8012f71..54981fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1690,13 +1690,18 @@ public class BlockManager {
namesystem.writeLock();
try {
for (int i = 0; i < timedOutItems.length; i++) {
+ /*
+ * Use the blockinfo from the blocksmap to be certain we're working
+ * with the most up-to-date block information (e.g. genstamp).
+ */
+ BlockInfoContiguous bi = blocksMap.getStoredBlock(timedOutItems[i]);
+ if (bi == null) {
+ continue;
+ }
NumberReplicas num = countNodes(timedOutItems[i]);
- if (isNeededReplication(timedOutItems[i], getReplication(timedOutItems[i]),
- num.liveReplicas())) {
- neededReplications.add(timedOutItems[i],
- num.liveReplicas(),
- num.decommissionedAndDecommissioning(),
- getReplication(timedOutItems[i]));
+ if (isNeededReplication(bi, getReplication(bi), num.liveReplicas())) {
+ neededReplications.add(bi, num.liveReplicas(),
+ num.decommissionedAndDecommissioning(), getReplication(bi));
}
}
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f2ba726/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
index c63badc..259404e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo.BlockStatus;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.junit.Test;
+import org.mockito.Mockito;
/**
* This class tests the internals of PendingReplicationBlocks.java,
@@ -52,13 +53,11 @@ public class TestPendingReplication {
private static final int DFS_REPLICATION_INTERVAL = 1;
// Number of datanodes in the cluster
private static final int DATANODE_COUNT = 5;
-
@Test
public void testPendingReplication() {
PendingReplicationBlocks pendingReplications;
pendingReplications = new PendingReplicationBlocks(TIMEOUT * 1000);
pendingReplications.start();
-
//
// Add 10 blocks to pendingReplications.
//
@@ -140,8 +139,7 @@ public class TestPendingReplication {
//
// Verify that everything has timed out.
//
- assertEquals("Size of pendingReplications ",
- 0, pendingReplications.size());
+ assertEquals("Size of pendingReplications ", 0, pendingReplications.size());
Block[] timedOut = pendingReplications.getTimedOutBlocks();
assertTrue(timedOut != null && timedOut.length == 15);
for (int i = 0; i < timedOut.length; i++) {
@@ -149,6 +147,98 @@ public class TestPendingReplication {
}
pendingReplications.stop();
}
+
+/* Test that processPendingReplications will use the most recent
+ * blockinfo from the blocksmap by placing a larger genstamp into
+ * the blocksmap.
+ */
+ @Test
+ public void testProcessPendingReplications() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ conf.setLong(
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY, TIMEOUT);
+ MiniDFSCluster cluster = null;
+ Block block;
+ BlockInfoContiguous blockInfo;
+ try {
+ cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(DATANODE_COUNT).build();
+ cluster.waitActive();
+
+ FSNamesystem fsn = cluster.getNamesystem();
+ BlockManager blkManager = fsn.getBlockManager();
+
+ PendingReplicationBlocks pendingReplications =
+ blkManager.pendingReplications;
+ UnderReplicatedBlocks neededReplications = blkManager.neededReplications;
+ BlocksMap blocksMap = blkManager.blocksMap;
+
+ //
+ // Add 1 block to pendingReplications with GenerationStamp = 0.
+ //
+
+ block = new Block(1, 1, 0);
+ blockInfo = new BlockInfoContiguous(block, (short) 3);
+
+ pendingReplications.increment(block,
+ DatanodeStorageInfo.toDatanodeDescriptors(
+ DFSTestUtil.createDatanodeStorageInfos(1)));
+ BlockCollection bc = Mockito.mock(BlockCollection.class);
+ Mockito.doReturn((short) 3).when(bc).getPreferredBlockReplication();
+ // Place into blocksmap with GenerationStamp = 1
+ blockInfo.setGenerationStamp(1);
+ blocksMap.addBlockCollection(blockInfo, bc);
+
+ assertEquals("Size of pendingReplications ", 1,
+ pendingReplications.size());
+
+ // Add a second block to pendingReplications that has no
+ // corresponding entry in blocksmap
+ block = new Block(2, 2, 0);
+ pendingReplications.increment(block,
+ DatanodeStorageInfo.toDatanodeDescriptors(
+ DFSTestUtil.createDatanodeStorageInfos(1)));
+
+ // verify 2 blocks in pendingReplications
+ assertEquals("Size of pendingReplications ", 2,
+ pendingReplications.size());
+
+ //
+ // Wait for everything to timeout.
+ //
+ while (pendingReplications.size() > 0) {
+ try {
+ Thread.sleep(100);
+ } catch (Exception e) {
+ }
+ }
+
+ //
+ // Verify that block moves to neededReplications
+ //
+ while (neededReplications.size() == 0) {
+ try {
+ Thread.sleep(100);
+ } catch (Exception e) {
+ }
+ }
+
+ // Verify that the generation stamp we will try to replicate
+ // is now 1
+ for (Block b: neededReplications) {
+ assertEquals("Generation stamp is 1 ", 1,
+ b.getGenerationStamp());
+ }
+
+ // Verify size of neededReplications is exactly 1.
+ assertEquals("size of neededReplications is 1 ", 1,
+ neededReplications.size());
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
/**
* Test if DatanodeProtocol#blockReceivedAndDeleted can correctly update the
[13/50] [abbrv] hadoop git commit: HDFS-8412. Fix the test failures
in HTTPFS: In some tests setReplication called after fs close. Contributed by
Uma Maheswara Rao G.
Posted by zj...@apache.org.
HDFS-8412. Fix the test failures in HTTPFS: In some tests setReplication called after fs close. Contributed by Uma Maheswara Rao G.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/01006c92
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/01006c92
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/01006c92
Branch: refs/heads/YARN-2928
Commit: 01006c92b9645e475c41b62ade9f2f66154952c8
Parents: e2e9de4
Author: Uma Maheswara Rao G <um...@apache.org>
Authored: Mon May 18 19:35:53 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:47:02 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01006c92/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
index 2cc67d4..0e082cc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
@@ -465,8 +465,8 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
OutputStream os = fs.create(path);
os.write(1);
os.close();
- fs.close();
fs.setReplication(path, (short) 2);
+ fs.close();
fs = getHttpFSFileSystem();
fs.setReplication(path, (short) 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01006c92/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3e0d360..8d0c5b6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -332,6 +332,8 @@ Trunk (Unreleased)
HDFS-8332. DFS client API calls should check filesystem closed (Rakesh R via umamahesh)
+ HDFS-8412. Fix the test failures in HTTPFS. (umamahesh)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
[21/50] [abbrv] hadoop git commit: Move HADOOP-11581 in CHANGES.txt
from 3.0.0 to 2.8.0.
Posted by zj...@apache.org.
Move HADOOP-11581 in CHANGES.txt from 3.0.0 to 2.8.0.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/84cd9647
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/84cd9647
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/84cd9647
Branch: refs/heads/YARN-2928
Commit: 84cd9647eea8e4faf7879a66b640993b2f41ce94
Parents: c6860e8
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue May 19 18:38:35 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:52:08 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84cd9647/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 8c7c978..baf9a0f 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -424,9 +424,6 @@ Trunk (Unreleased)
HADOOP-11559. Add links to RackAwareness and InterfaceClassification
to site index (Masatake Iwasaki via aw)
- HADOOP-11581. Multithreaded correctness Warnings
- #org.apache.hadoop.fs.shell.Ls (Brahma Reddy Battula via aw)
-
HADOOP-11580. Remove SingleNodeSetup.md from trunk (aajisaka)
HADOOP-11583. Fix syntax error in SecureMode.md (Masatake Iwasaki via aw)
@@ -713,6 +710,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-10582. Fix the test case for copying to non-existent dir in
TestFsShellCopy. (Kousuke Saruta via aajisaka)
+ HADOOP-11581. Multithreaded correctness Warnings
+ #org.apache.hadoop.fs.shell.Ls (Brahma Reddy Battula via aw)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
[43/50] [abbrv] hadoop git commit: YARN-3609. Load node labels from
storage inside RM serviceStart. Contributed by Wangda Tan
Posted by zj...@apache.org.
YARN-3609. Load node labels from storage inside RM serviceStart. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/64074a35
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/64074a35
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/64074a35
Branch: refs/heads/YARN-2928
Commit: 64074a358cd2fda775b4c2edac1137250c14ed42
Parents: f416078
Author: Jian He <ji...@apache.org>
Authored: Wed May 20 16:26:02 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 12:42:57 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../nodelabels/CommonNodeLabelsManager.java | 8 +-
.../TestFileSystemNodeLabelsStore.java | 4 +
.../yarn/server/resourcemanager/MockRM.java | 26 ++++--
.../server/resourcemanager/RMHATestBase.java | 4 +-
.../resourcemanager/TestRMHAForNodeLabels.java | 93 ++++++++++++++++++++
6 files changed, 125 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64074a35/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f1b5a2d..fa796ac 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -610,6 +610,9 @@ Release 2.7.1 - UNRELEASED
YARN-3681. yarn cmd says "could not find main class 'queue'" in windows.
(Craig Welch and Varun Saxena via xgong)
+ YARN-3609. Load node labels from storage inside RM serviceStart. (Wangda
+ Tan via jianhe)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64074a35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index badf4d6..2d57ad6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -222,10 +222,6 @@ public class CommonNodeLabelsManager extends AbstractService {
isDistributedNodeLabelConfiguration =
YarnConfiguration.isDistributedNodeLabelConfiguration(conf);
-
- if (nodeLabelsEnabled) {
- initNodeLabelStore(conf);
- }
labelCollections.put(NO_LABEL, new RMNodeLabel(NO_LABEL));
}
@@ -245,6 +241,10 @@ public class CommonNodeLabelsManager extends AbstractService {
@Override
protected void serviceStart() throws Exception {
+ if (nodeLabelsEnabled) {
+ initNodeLabelStore(getConfig());
+ }
+
// init dispatcher only when service start, because recover will happen in
// service init, we don't want to trigger any event handling at that time.
initDispatcher(getConfig());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64074a35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
index fb60cd6..4b052c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
@@ -108,6 +108,7 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
mgr = new MockNodeLabelManager();
mgr.init(conf);
+ mgr.start();
// check variables
Assert.assertEquals(3, mgr.getClusterNodeLabelNames().size());
@@ -127,6 +128,7 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
mgr.stop();
mgr = new MockNodeLabelManager();
mgr.init(conf);
+ mgr.start();
// check variables
Assert.assertEquals(3, mgr.getClusterNodeLabelNames().size());
@@ -165,6 +167,7 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
cf.set(YarnConfiguration.NODELABEL_CONFIGURATION_TYPE,
YarnConfiguration.DISTRIBUTED_NODELABEL_CONFIGURATION_TYPE);
mgr.init(cf);
+ mgr.start();
// check variables
Assert.assertEquals(3, mgr.getClusterNodeLabels().size());
@@ -205,6 +208,7 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
mgr = new MockNodeLabelManager();
mgr.init(conf);
+ mgr.start();
// check variables
Assert.assertEquals(3, mgr.getClusterNodeLabelNames().size());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64074a35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 3469b0c..672ce13 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -65,7 +65,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabels
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
@@ -98,6 +97,8 @@ public class MockRM extends ResourceManager {
static final Logger LOG = Logger.getLogger(MockRM.class);
static final String ENABLE_WEBAPP = "mockrm.webapp.enabled";
+
+ final private boolean useNullRMNodeLabelsManager;
public MockRM() {
this(new YarnConfiguration());
@@ -108,20 +109,31 @@ public class MockRM extends ResourceManager {
}
public MockRM(Configuration conf, RMStateStore store) {
- super();
+ this(conf, store, true);
+ }
+
+ public MockRM(Configuration conf, RMStateStore store,
+ boolean useNullRMNodeLabelsManager) {
+ super();
+ this.useNullRMNodeLabelsManager = useNullRMNodeLabelsManager;
init(conf instanceof YarnConfiguration ? conf : new YarnConfiguration(conf));
if(store != null) {
setRMStateStore(store);
}
Logger rootLogger = LogManager.getRootLogger();
- rootLogger.setLevel(Level.DEBUG);
+ rootLogger.setLevel(Level.DEBUG);
}
@Override
- protected RMNodeLabelsManager createNodeLabelManager() {
- RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
- mgr.init(getConfig());
- return mgr;
+ protected RMNodeLabelsManager createNodeLabelManager()
+ throws InstantiationException, IllegalAccessException {
+ if (useNullRMNodeLabelsManager) {
+ RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
+ mgr.init(getConfig());
+ return mgr;
+ } else {
+ return super.createNodeLabelManager();
+ }
}
public void waitForState(ApplicationId appId, RMAppState finalState)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64074a35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java
index 9f54de8..6f3666f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java
@@ -106,8 +106,8 @@ public class RMHATestBase extends ClientBaseWithFixes{
}
protected void startRMs() throws IOException {
- rm1 = new MockRM(confForRM1);
- rm2 = new MockRM(confForRM2);
+ rm1 = new MockRM(confForRM1, null, false);
+ rm2 = new MockRM(confForRM2, null, false);
startRMs(rm1, confForRM1, rm2, confForRM2);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64074a35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHAForNodeLabels.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHAForNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHAForNodeLabels.java
new file mode 100644
index 0000000..25d9c56
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHAForNodeLabels.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableSet;
+
+public class TestRMHAForNodeLabels extends RMHATestBase {
+ public static final Log LOG = LogFactory
+ .getLog(TestSubmitApplicationWithRMHA.class);
+
+ @Before
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+
+ // Create directory for node label store
+ File tempDir = File.createTempFile("nlb", ".tmp");
+ tempDir.delete();
+ tempDir.mkdirs();
+ tempDir.deleteOnExit();
+
+ confForRM1.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+ confForRM1.set(YarnConfiguration.FS_NODE_LABELS_STORE_ROOT_DIR,
+ tempDir.getAbsolutePath());
+
+ confForRM2.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+ confForRM2.set(YarnConfiguration.FS_NODE_LABELS_STORE_ROOT_DIR,
+ tempDir.getAbsolutePath());
+ }
+
+ @Test
+ public void testRMHARecoverNodeLabels() throws Exception {
+ // start two RMs, and transit rm1 to active, rm2 to standby
+ startRMs();
+
+ // Add labels to rm1
+ rm1.getRMContext()
+ .getNodeLabelManager()
+ .addToCluserNodeLabels(
+ Arrays.asList(NodeLabel.newInstance("a"),
+ NodeLabel.newInstance("b"), NodeLabel.newInstance("c")));
+
+ Map<NodeId, Set<String>> nodeToLabels = new HashMap<>();
+ nodeToLabels.put(NodeId.newInstance("host1", 0), ImmutableSet.of("a"));
+ nodeToLabels.put(NodeId.newInstance("host2", 0), ImmutableSet.of("b"));
+
+ rm1.getRMContext().getNodeLabelManager().replaceLabelsOnNode(nodeToLabels);
+
+ // Do the failover
+ explicitFailover();
+
+ // Check labels in rm2
+ Assert
+ .assertTrue(rm2.getRMContext().getNodeLabelManager()
+ .getClusterNodeLabelNames()
+ .containsAll(ImmutableSet.of("a", "b", "c")));
+ Assert.assertTrue(rm2.getRMContext().getNodeLabelManager()
+ .getNodeLabels().get(NodeId.newInstance("host1", 0)).contains("a"));
+ Assert.assertTrue(rm2.getRMContext().getNodeLabelManager()
+ .getNodeLabels().get(NodeId.newInstance("host2", 0)).contains("b"));
+ }
+}
[22/50] [abbrv] hadoop git commit: HADOOP-11103. Clean up
RemoteException (Contributed by Sean Busbey)
Posted by zj...@apache.org.
HADOOP-11103. Clean up RemoteException (Contributed by Sean Busbey)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c6860e8e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c6860e8e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c6860e8e
Branch: refs/heads/YARN-2928
Commit: c6860e8e97f8cd4397f62a1aee6430b1385e3732
Parents: 5ab2361
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue May 19 14:41:05 2015 +0530
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:52:08 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../main/java/org/apache/hadoop/ipc/Client.java | 5 +--
.../org/apache/hadoop/ipc/RemoteException.java | 32 +++++++++++++++-----
.../java/org/apache/hadoop/ipc/TestRPC.java | 4 +++
.../hdfs/server/namenode/ha/TestHASafeMode.java | 3 ++
5 files changed, 35 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6860e8e/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index ee7d1e3..8c7c978 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -588,6 +588,8 @@ Release 2.8.0 - UNRELEASED
HADOOP-1540. Support file exclusion list in distcp. (Rich Haase via jing9)
+ HADOOP-11103. Clean up RemoteException (Sean Busbey via vinayakumarb)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6860e8e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index 97b715b..f28d8a2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -1139,10 +1139,7 @@ public class Client {
if (erCode == null) {
LOG.warn("Detailed error code not set by server on rpc error");
}
- RemoteException re =
- ( (erCode == null) ?
- new RemoteException(exceptionClassName, errorMsg) :
- new RemoteException(exceptionClassName, errorMsg, erCode));
+ RemoteException re = new RemoteException(exceptionClassName, errorMsg, erCode);
if (status == RpcStatusProto.ERROR) {
calls.remove(callId);
call.setException(re);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6860e8e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java
index 7926d86..620e100 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RemoteException.java
@@ -25,31 +25,46 @@ import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.Rpc
import org.xml.sax.Attributes;
public class RemoteException extends IOException {
+ /** this value should not be defined in RpcHeader.proto so that protobuf will return a null */
+ private static final int UNSPECIFIED_ERROR = -1;
/** For java.io.Serializable */
private static final long serialVersionUID = 1L;
private final int errorCode;
- private String className;
+ private final String className;
+ /**
+ * @param className wrapped exception, may be null
+ * @param msg may be null
+ */
public RemoteException(String className, String msg) {
- super(msg);
- this.className = className;
- errorCode = -1;
+ this(className, msg, null);
}
+ /**
+ * @param className wrapped exception, may be null
+ * @param msg may be null
+ * @param erCode may be null
+ */
public RemoteException(String className, String msg, RpcErrorCodeProto erCode) {
super(msg);
this.className = className;
if (erCode != null)
errorCode = erCode.getNumber();
else
- errorCode = -1;
+ errorCode = UNSPECIFIED_ERROR;
}
+ /**
+ * @return the class name for the wrapped exception; may be null if none was given.
+ */
public String getClassName() {
return className;
}
+ /**
+ * @return may be null if the code was newer than our protobuf definitions or none was given.
+ */
public RpcErrorCodeProto getErrorCode() {
return RpcErrorCodeProto.valueOf(errorCode);
}
@@ -60,7 +75,7 @@ public class RemoteException extends IOException {
* <p>
* Unwraps any IOException.
*
- * @param lookupTypes the desired exception class.
+ * @param lookupTypes the desired exception class. may be null.
* @return IOException, which is either the lookupClass exception or this.
*/
public IOException unwrapRemoteException(Class<?>... lookupTypes) {
@@ -108,7 +123,10 @@ public class RemoteException extends IOException {
return ex;
}
- /** Create RemoteException from attributes */
+ /**
+ * Create RemoteException from attributes
+ * @param attrs may not be null
+ */
public static RemoteException valueOf(Attributes attrs) {
return new RemoteException(attrs.getValue("class"),
attrs.getValue("message"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6860e8e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
index f049395..d36a671 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.io.retry.RetryProxy;
import org.apache.hadoop.ipc.Client.ConnectionId;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcErrorCodeProto;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
@@ -589,6 +590,7 @@ public class TestRPC {
}
} catch (RemoteException e) {
if (expectFailure) {
+ assertEquals("RPC error code should be UNAUTHORIZED", RpcErrorCodeProto.FATAL_UNAUTHORIZED, e.getErrorCode());
assertTrue(e.unwrapRemoteException() instanceof AuthorizationException);
} else {
throw e;
@@ -728,6 +730,7 @@ public class TestRPC {
proxy.echo("");
} catch (RemoteException e) {
LOG.info("LOGGING MESSAGE: " + e.getLocalizedMessage());
+ assertEquals("RPC error code should be UNAUTHORIZED", RpcErrorCodeProto.FATAL_UNAUTHORIZED, e.getErrorCode());
assertTrue(e.unwrapRemoteException() instanceof AccessControlException);
succeeded = true;
} finally {
@@ -757,6 +760,7 @@ public class TestRPC {
proxy.echo("");
} catch (RemoteException e) {
LOG.info("LOGGING MESSAGE: " + e.getLocalizedMessage());
+ assertEquals("RPC error code should be UNAUTHORIZED", RpcErrorCodeProto.FATAL_UNAUTHORIZED, e.getErrorCode());
assertTrue(e.unwrapRemoteException() instanceof AccessControlException);
succeeded = true;
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6860e8e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
index 86f3e7b..9ded0ed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.StandbyException;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcErrorCodeProto;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
import org.junit.After;
@@ -774,6 +775,8 @@ public class TestHASafeMode {
fail("StandBy should throw exception for isInSafeMode");
} catch (IOException e) {
if (e instanceof RemoteException) {
+ assertEquals("RPC Error code should indicate app failure.", RpcErrorCodeProto.ERROR_APPLICATION,
+ ((RemoteException) e).getErrorCode());
IOException sbExcpetion = ((RemoteException) e).unwrapRemoteException();
assertTrue("StandBy nn should not support isInSafeMode",
sbExcpetion instanceof StandbyException);
[28/50] [abbrv] hadoop git commit: YARN-3302.
TestDockerContainerExecutor should run automatically if it can detect docker
in the usual place (Ravindra Kumar Naik via raviprak)
Posted by zj...@apache.org.
YARN-3302. TestDockerContainerExecutor should run automatically if it can detect docker in the usual place (Ravindra Kumar Naik via raviprak)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/509e6cbe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/509e6cbe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/509e6cbe
Branch: refs/heads/YARN-2928
Commit: 509e6cbe56f56a52ef1cb9fc66e94cfcaff8e96a
Parents: b22cb5e
Author: Ravi Prakash <ra...@altiscale.com>
Authored: Tue May 19 10:28:11 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:52:09 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../TestDockerContainerExecutor.java | 27 +++++++++++++++-----
2 files changed, 24 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/509e6cbe/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a2f3914..e5a0b95 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -498,6 +498,9 @@ Release 2.8.0 - UNRELEASED
YARN-2421. RM still allocates containers to an app in the FINISHING
state (Chang Li via jlowe)
+ YARN-3302. TestDockerContainerExecutor should run automatically if it can
+ detect docker in the usual place (Ravindra Kumar Naik via raviprak)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/509e6cbe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutor.java
index 65e381c..9386897 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutor.java
@@ -51,10 +51,11 @@ import com.google.common.base.Strings;
* This is intended to test the DockerContainerExecutor code, but it requires
* docker to be installed.
* <br><ol>
- * <li>Install docker, and Compile the code with docker-service-url set to the
- * host and port where docker service is running.
+ * <li>To run the tests, set the docker-service-url to the host and port where
+ * docker service is running (If docker-service-url is not specified then the
+ * local daemon will be used).
* <br><pre><code>
- * > mvn clean install -Ddocker-service-url=tcp://0.0.0.0:4243 -DskipTests
+ * mvn test -Ddocker-service-url=tcp://0.0.0.0:4243 -Dtest=TestDockerContainerExecutor
* </code></pre>
*/
public class TestDockerContainerExecutor {
@@ -98,10 +99,13 @@ public class TestDockerContainerExecutor {
dockerUrl = System.getProperty("docker-service-url");
LOG.info("dockerUrl: " + dockerUrl);
- if (Strings.isNullOrEmpty(dockerUrl)) {
+ if (!Strings.isNullOrEmpty(dockerUrl)) {
+ dockerUrl = " -H " + dockerUrl;
+ } else if(isDockerDaemonRunningLocally()) {
+ dockerUrl = "";
+ } else {
return;
}
- dockerUrl = " -H " + dockerUrl;
dockerExec = "docker " + dockerUrl;
conf.set(
YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, yarnImage);
@@ -136,6 +140,17 @@ public class TestDockerContainerExecutor {
return exec != null;
}
+ private boolean isDockerDaemonRunningLocally() {
+ boolean dockerDaemonRunningLocally = true;
+ try {
+ shellExec("docker info");
+ } catch (Exception e) {
+ LOG.info("docker daemon is not running on local machine.");
+ dockerDaemonRunningLocally = false;
+ }
+ return dockerDaemonRunningLocally;
+ }
+
/**
* Test that a docker container can be launched to run a command
* @param cId a fake ContainerID
@@ -200,7 +215,7 @@ public class TestDockerContainerExecutor {
* Test that a touch command can be launched successfully in a docker
* container
*/
- @Test
+ @Test(timeout=1000000)
public void testLaunchContainer() throws IOException {
if (!shouldRun()) {
LOG.warn("Docker not installed, aborting test.");
[09/50] [abbrv] hadoop git commit: HDFS-8157. Writes to RAM DISK
reserve locked memory for block files. (Arpit Agarwal)
Posted by zj...@apache.org.
HDFS-8157. Writes to RAM DISK reserve locked memory for block files. (Arpit Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d01afb2a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d01afb2a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d01afb2a
Branch: refs/heads/YARN-2928
Commit: d01afb2a943ac566175b20ba9434b5b4116a7351
Parents: 7f53ed0
Author: Arpit Agarwal <ar...@apache.org>
Authored: Sat May 16 09:05:35 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:47:01 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/datanode/ReplicaInPipeline.java | 11 +-
.../hdfs/server/datanode/ReplicaInfo.java | 12 +-
.../server/datanode/fsdataset/FsVolumeSpi.java | 8 +
.../datanode/fsdataset/impl/BlockPoolSlice.java | 2 +-
.../impl/FsDatasetAsyncDiskService.java | 7 +-
.../datanode/fsdataset/impl/FsDatasetCache.java | 85 +++++++-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 106 ++++++----
.../datanode/fsdataset/impl/FsVolumeImpl.java | 20 +-
.../impl/RamDiskReplicaLruTracker.java | 19 +-
.../fsdataset/impl/RamDiskReplicaTracker.java | 12 +-
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 2 +-
.../hdfs/server/balancer/TestBalancer.java | 9 +-
.../server/datanode/SimulatedFSDataset.java | 4 +
.../server/datanode/TestDirectoryScanner.java | 9 +
.../server/datanode/TestFsDatasetCache.java | 4 +-
.../datanode/extdataset/ExternalVolumeImpl.java | 4 +
.../fsdataset/impl/LazyPersistTestCase.java | 57 ++++--
.../impl/TestLazyPersistLockedMemory.java | 201 +++++++++++++++++++
.../fsdataset/impl/TestWriteToReplica.java | 4 +-
20 files changed, 497 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d01afb2a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0c4d850..8c823ef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -560,6 +560,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8394. Move getAdditionalBlock() and related functionalities into a
separate class. (wheat9)
+ HDFS-8157. Writes to RAM DISK reserve locked memory for block files.
+ (Arpit Agarwal)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d01afb2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
index cc55f85..0eb143a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInPipeline.java
@@ -51,7 +51,8 @@ public class ReplicaInPipeline extends ReplicaInfo
* the bytes already written to this block.
*/
private long bytesReserved;
-
+ private final long originalBytesReserved;
+
/**
* Constructor for a zero length replica
* @param blockId block id
@@ -97,6 +98,7 @@ public class ReplicaInPipeline extends ReplicaInfo
this.bytesOnDisk = len;
this.writer = writer;
this.bytesReserved = bytesToReserve;
+ this.originalBytesReserved = bytesToReserve;
}
/**
@@ -109,6 +111,7 @@ public class ReplicaInPipeline extends ReplicaInfo
this.bytesOnDisk = from.getBytesOnDisk();
this.writer = from.writer;
this.bytesReserved = from.bytesReserved;
+ this.originalBytesReserved = from.originalBytesReserved;
}
@Override
@@ -149,8 +152,14 @@ public class ReplicaInPipeline extends ReplicaInfo
}
@Override
+ public long getOriginalBytesReserved() {
+ return originalBytesReserved;
+ }
+
+ @Override
public void releaseAllBytesReserved() { // ReplicaInPipelineInterface
getVolume().releaseReservedSpace(bytesReserved);
+ getVolume().releaseLockedMemory(bytesReserved);
bytesReserved = 0;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d01afb2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
index 940d3eb..136d8a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
@@ -218,7 +218,17 @@ abstract public class ReplicaInfo extends Block implements Replica {
public long getBytesReserved() {
return 0;
}
-
+
+ /**
+ * Number of bytes originally reserved for this replica. The actual
+ * reservation is adjusted as data is written to disk.
+ *
+ * @return the number of bytes originally reserved for this replica.
+ */
+ public long getOriginalBytesReserved() {
+ return 0;
+ }
+
/**
* Copy specified file into a temporary file. Then rename the
* temporary file to the original name. This will cause any
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d01afb2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
index 2a8f31b..8d1bb2a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java
@@ -73,6 +73,14 @@ public interface FsVolumeSpi {
public void releaseReservedSpace(long bytesToRelease);
/**
+ * Release reserved memory for an RBW block written to transient storage
+ * i.e. RAM.
+ * bytesToRelease will be rounded down to the OS page size since locked
+ * memory reservation must always be a multiple of the page size.
+ */
+ public void releaseLockedMemory(long bytesToRelease);
+
+ /**
* BlockIterator will return ExtendedBlock entries from a block pool in
* this volume. The entries will be returned in sorted order.<p/>
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d01afb2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index a47d564..951c759 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -475,7 +475,7 @@ class BlockPoolSlice {
// eventually.
if (newReplica.getVolume().isTransientStorage()) {
lazyWriteReplicaMap.addReplica(bpid, blockId,
- (FsVolumeImpl) newReplica.getVolume());
+ (FsVolumeImpl) newReplica.getVolume(), 0);
} else {
lazyWriteReplicaMap.discardReplica(bpid, blockId, false);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d01afb2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
index c1d3990..fdc9f83 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
import java.io.File;
import java.io.FileDescriptor;
-import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@@ -277,7 +276,8 @@ class FsDatasetAsyncDiskService {
@Override
public void run() {
- long dfsBytes = blockFile.length() + metaFile.length();
+ final long blockLength = blockFile.length();
+ final long metaLength = metaFile.length();
boolean result;
result = (trashDirectory == null) ? deleteFiles() : moveFiles();
@@ -291,7 +291,8 @@ class FsDatasetAsyncDiskService {
if(block.getLocalBlock().getNumBytes() != BlockCommand.NO_ACK){
datanode.notifyNamenodeDeletedBlock(block, volume.getStorageID());
}
- volume.decDfsUsed(block.getBlockPoolId(), dfsBytes);
+ volume.onBlockFileDeletion(block.getBlockPoolId(), blockLength);
+ volume.onMetaFileDeletion(block.getBlockPoolId(), metaLength);
LOG.info("Deleted " + block.getBlockPoolId() + " "
+ block.getLocalBlock() + " file " + blockFile);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d01afb2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java
index e0df0f2..6f524b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java
@@ -151,10 +151,15 @@ public class FsDatasetCache {
/**
* Round up a number to the operating system page size.
*/
- public long round(long count) {
- long newCount =
- (count + (osPageSize - 1)) / osPageSize;
- return newCount * osPageSize;
+ public long roundUp(long count) {
+ return (count + osPageSize - 1) & (~(osPageSize - 1));
+ }
+
+ /**
+ * Round down a number to the operating system page size.
+ */
+ public long roundDown(long count) {
+ return count & (~(osPageSize - 1));
}
}
@@ -173,7 +178,7 @@ public class FsDatasetCache {
* -1 if we failed.
*/
long reserve(long count) {
- count = rounder.round(count);
+ count = rounder.roundUp(count);
while (true) {
long cur = usedBytes.get();
long next = cur + count;
@@ -195,10 +200,23 @@ public class FsDatasetCache {
* @return The new number of usedBytes.
*/
long release(long count) {
- count = rounder.round(count);
+ count = rounder.roundUp(count);
return usedBytes.addAndGet(-count);
}
-
+
+ /**
+ * Release some bytes that we're using rounded down to the page size.
+ *
+ * @param count The number of bytes to release. We will round this
+ * down to the page size.
+ *
+ * @return The new number of usedBytes.
+ */
+ long releaseRoundDown(long count) {
+ count = rounder.roundDown(count);
+ return usedBytes.addAndGet(-count);
+ }
+
long get() {
return usedBytes.get();
}
@@ -341,6 +359,52 @@ public class FsDatasetCache {
}
/**
+ * Try to reserve more bytes.
+ *
+ * @param count The number of bytes to add. We will round this
+ * up to the page size.
+ *
+ * @return The new number of usedBytes if we succeeded;
+ * -1 if we failed.
+ */
+ long reserve(long count) {
+ return usedBytesCount.reserve(count);
+ }
+
+ /**
+ * Release some bytes that we're using.
+ *
+ * @param count The number of bytes to release. We will round this
+ * up to the page size.
+ *
+ * @return The new number of usedBytes.
+ */
+ long release(long count) {
+ return usedBytesCount.release(count);
+ }
+
+ /**
+ * Release some bytes that we're using rounded down to the page size.
+ *
+ * @param count The number of bytes to release. We will round this
+ * down to the page size.
+ *
+ * @return The new number of usedBytes.
+ */
+ long releaseRoundDown(long count) {
+ return usedBytesCount.releaseRoundDown(count);
+ }
+
+ /**
+ * Get the OS page size.
+ *
+ * @return the OS page size.
+ */
+ long getOsPageSize() {
+ return usedBytesCount.rounder.osPageSize;
+ }
+
+ /**
* Background worker that mmaps, mlocks, and checksums a block
*/
private class CachingTask implements Runnable {
@@ -363,7 +427,7 @@ public class FsDatasetCache {
MappableBlock mappableBlock = null;
ExtendedBlock extBlk = new ExtendedBlock(key.getBlockPoolId(),
key.getBlockId(), length, genstamp);
- long newUsedBytes = usedBytesCount.reserve(length);
+ long newUsedBytes = reserve(length);
boolean reservedBytes = false;
try {
if (newUsedBytes < 0) {
@@ -423,7 +487,7 @@ public class FsDatasetCache {
IOUtils.closeQuietly(metaIn);
if (!success) {
if (reservedBytes) {
- usedBytesCount.release(length);
+ release(length);
}
LOG.debug("Caching of {} was aborted. We are now caching only {} "
+ "bytes in total.", key, usedBytesCount.get());
@@ -502,8 +566,7 @@ public class FsDatasetCache {
synchronized (FsDatasetCache.this) {
mappableBlockMap.remove(key);
}
- long newUsedBytes =
- usedBytesCount.release(value.mappableBlock.getLength());
+ long newUsedBytes = release(value.mappableBlock.getLength());
numBlocksCached.addAndGet(-1);
dataset.datanode.getMetrics().incrBlocksUncached(1);
if (revocationTimeMs != 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d01afb2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 8725126..8ebd214 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -319,8 +319,18 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
cacheManager = new FsDatasetCache(this);
// Start the lazy writer once we have built the replica maps.
- lazyWriter = new Daemon(new LazyWriter(conf));
- lazyWriter.start();
+ // We need to start the lazy writer even if MaxLockedMemory is set to
+ // zero because we may have un-persisted replicas in memory from before
+ // the process restart. To minimize the chances of data loss we'll
+ // ensure they get written to disk now.
+ if (ramDiskReplicaTracker.numReplicasNotPersisted() > 0 ||
+ datanode.getDnConf().getMaxLockedMemory() > 0) {
+ lazyWriter = new Daemon(new LazyWriter(conf));
+ lazyWriter.start();
+ } else {
+ lazyWriter = null;
+ }
+
registerMBean(datanode.getDatanodeUuid());
// Add a Metrics2 Source Interface. This is same
@@ -1284,26 +1294,33 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
" and thus cannot be created.");
}
// create a new block
- FsVolumeReference ref;
- while (true) {
+ FsVolumeReference ref = null;
+
+ // Use ramdisk only if block size is a multiple of OS page size.
+ // This simplifies reservation for partially used replicas
+ // significantly.
+ if (allowLazyPersist &&
+ lazyWriter != null &&
+ b.getNumBytes() % cacheManager.getOsPageSize() == 0 &&
+ (cacheManager.reserve(b.getNumBytes())) > 0) {
try {
- if (allowLazyPersist) {
- // First try to place the block on a transient volume.
- ref = volumes.getNextTransientVolume(b.getNumBytes());
- datanode.getMetrics().incrRamDiskBlocksWrite();
- } else {
- ref = volumes.getNextVolume(storageType, b.getNumBytes());
- }
- } catch (DiskOutOfSpaceException de) {
- if (allowLazyPersist) {
- datanode.getMetrics().incrRamDiskBlocksWriteFallback();
- allowLazyPersist = false;
- continue;
+ // First try to place the block on a transient volume.
+ ref = volumes.getNextTransientVolume(b.getNumBytes());
+ datanode.getMetrics().incrRamDiskBlocksWrite();
+ } catch(DiskOutOfSpaceException de) {
+ // Ignore the exception since we just fall back to persistent storage.
+ datanode.getMetrics().incrRamDiskBlocksWriteFallback();
+ } finally {
+ if (ref == null) {
+ cacheManager.release(b.getNumBytes());
}
- throw de;
}
- break;
}
+
+ if (ref == null) {
+ ref = volumes.getNextVolume(storageType, b.getNumBytes());
+ }
+
FsVolumeImpl v = (FsVolumeImpl) ref.getVolume();
// create an rbw file to hold block in the designated volume
File f;
@@ -1564,7 +1581,11 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
newReplicaInfo = new FinalizedReplica(replicaInfo, v, dest.getParentFile());
if (v.isTransientStorage()) {
- ramDiskReplicaTracker.addReplica(bpid, replicaInfo.getBlockId(), v);
+ releaseLockedMemory(
+ replicaInfo.getOriginalBytesReserved() - replicaInfo.getNumBytes(),
+ false);
+ ramDiskReplicaTracker.addReplica(
+ bpid, replicaInfo.getBlockId(), v, replicaInfo.getNumBytes());
datanode.getMetrics().addRamDiskBytesWrite(replicaInfo.getNumBytes());
}
}
@@ -1811,9 +1832,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
/**
- * We're informed that a block is no longer valid. We
- * could lazily garbage-collect the block, but why bother?
- * just get rid of it.
+ * We're informed that a block is no longer valid. Delete it.
*/
@Override // FsDatasetSpi
public void invalidate(String bpid, Block invalidBlks[]) throws IOException {
@@ -2064,8 +2083,10 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
public void shutdown() {
fsRunning = false;
- ((LazyWriter) lazyWriter.getRunnable()).stop();
- lazyWriter.interrupt();
+ if (lazyWriter != null) {
+ ((LazyWriter) lazyWriter.getRunnable()).stop();
+ lazyWriter.interrupt();
+ }
if (mbeanName != null) {
MBeans.unregister(mbeanName);
@@ -2083,11 +2104,13 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
volumes.shutdown();
}
- try {
- lazyWriter.join();
- } catch (InterruptedException ie) {
- LOG.warn("FsDatasetImpl.shutdown ignoring InterruptedException " +
- "from LazyWriter.join");
+ if (lazyWriter != null) {
+ try {
+ lazyWriter.join();
+ } catch (InterruptedException ie) {
+ LOG.warn("FsDatasetImpl.shutdown ignoring InterruptedException " +
+ "from LazyWriter.join");
+ }
}
}
@@ -2173,7 +2196,11 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
diskFile.length(), diskGS, vol, diskFile.getParentFile());
volumeMap.add(bpid, diskBlockInfo);
if (vol.isTransientStorage()) {
- ramDiskReplicaTracker.addReplica(bpid, blockId, (FsVolumeImpl) vol);
+ long lockedBytesReserved =
+ cacheManager.reserve(diskBlockInfo.getNumBytes()) > 0 ?
+ diskBlockInfo.getNumBytes() : 0;
+ ramDiskReplicaTracker.addReplica(
+ bpid, blockId, (FsVolumeImpl) vol, lockedBytesReserved);
}
LOG.warn("Added missing block to memory " + diskBlockInfo);
return;
@@ -2760,12 +2787,14 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
boolean ramDiskConfigured = ramDiskConfigured();
// Add thread for DISK volume if RamDisk is configured
if (ramDiskConfigured &&
+ asyncLazyPersistService != null &&
!asyncLazyPersistService.queryVolume(v.getCurrentDir())) {
asyncLazyPersistService.addVolume(v.getCurrentDir());
}
// Remove thread for DISK volume if RamDisk is not configured
if (!ramDiskConfigured &&
+ asyncLazyPersistService != null &&
asyncLazyPersistService.queryVolume(v.getCurrentDir())) {
asyncLazyPersistService.removeVolume(v.getCurrentDir());
}
@@ -2790,9 +2819,10 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
// Remove the old replicas
if (blockFile.delete() || !blockFile.exists()) {
- ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(bpid, blockFileUsed);
+ FsVolumeImpl volume = (FsVolumeImpl) replicaInfo.getVolume();
+ volume.onBlockFileDeletion(bpid, blockFileUsed);
if (metaFile.delete() || !metaFile.exists()) {
- ((FsVolumeImpl) replicaInfo.getVolume()).decDfsUsed(bpid, metaFileUsed);
+ volume.onMetaFileDeletion(bpid, metaFileUsed);
}
}
@@ -2905,8 +2935,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
/**
- * Attempt to evict one or more transient block replicas we have at least
- * spaceNeeded bytes free.
+ * Attempt to evict one or more transient block replicas until we
+ * have at least spaceNeeded bytes free.
*/
private void evictBlocks() throws IOException {
int iterations = 0;
@@ -3056,5 +3086,13 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
s.add(blockId);
}
}
+
+ void releaseLockedMemory(long count, boolean roundup) {
+ if (roundup) {
+ cacheManager.release(count);
+ } else {
+ cacheManager.releaseRoundDown(count);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d01afb2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index bc96a02..49a56bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -274,7 +274,18 @@ public class FsVolumeImpl implements FsVolumeSpi {
return getBlockPoolSlice(bpid).getTmpDir();
}
- void decDfsUsed(String bpid, long value) {
+ void onBlockFileDeletion(String bpid, long value) {
+ decDfsUsed(bpid, value);
+ if (isTransientStorage()) {
+ dataset.releaseLockedMemory(value, true);
+ }
+ }
+
+ void onMetaFileDeletion(String bpid, long value) {
+ decDfsUsed(bpid, value);
+ }
+
+ private void decDfsUsed(String bpid, long value) {
synchronized(dataset) {
BlockPoolSlice bp = bpSlices.get(bpid);
if (bp != null) {
@@ -428,6 +439,13 @@ public class FsVolumeImpl implements FsVolumeSpi {
}
}
+ @Override
+ public void releaseLockedMemory(long bytesToRelease) {
+ if (isTransientStorage()) {
+ dataset.releaseLockedMemory(bytesToRelease, false);
+ }
+ }
+
private enum SubdirFilter implements FilenameFilter {
INSTANCE;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d01afb2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java
index c01a6cf..b940736 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaLruTracker.java
@@ -38,8 +38,10 @@ public class RamDiskReplicaLruTracker extends RamDiskReplicaTracker {
private class RamDiskReplicaLru extends RamDiskReplica {
long lastUsedTime;
- private RamDiskReplicaLru(String bpid, long blockId, FsVolumeImpl ramDiskVolume) {
- super(bpid, blockId, ramDiskVolume);
+ private RamDiskReplicaLru(String bpid, long blockId,
+ FsVolumeImpl ramDiskVolume,
+ long lockedBytesReserved) {
+ super(bpid, blockId, ramDiskVolume, lockedBytesReserved);
}
@Override
@@ -70,20 +72,23 @@ public class RamDiskReplicaLruTracker extends RamDiskReplicaTracker {
TreeMultimap<Long, RamDiskReplicaLru> replicasPersisted;
RamDiskReplicaLruTracker() {
- replicaMaps = new HashMap<String, Map<Long, RamDiskReplicaLru>>();
- replicasNotPersisted = new LinkedList<RamDiskReplicaLru>();
+ replicaMaps = new HashMap<>();
+ replicasNotPersisted = new LinkedList<>();
replicasPersisted = TreeMultimap.create();
}
@Override
synchronized void addReplica(final String bpid, final long blockId,
- final FsVolumeImpl transientVolume) {
+ final FsVolumeImpl transientVolume,
+ long lockedBytesReserved) {
Map<Long, RamDiskReplicaLru> map = replicaMaps.get(bpid);
if (map == null) {
- map = new HashMap<Long, RamDiskReplicaLru>();
+ map = new HashMap<>();
replicaMaps.put(bpid, map);
}
- RamDiskReplicaLru ramDiskReplicaLru = new RamDiskReplicaLru(bpid, blockId, transientVolume);
+ RamDiskReplicaLru ramDiskReplicaLru =
+ new RamDiskReplicaLru(bpid, blockId, transientVolume,
+ lockedBytesReserved);
map.put(blockId, ramDiskReplicaLru);
replicasNotPersisted.add(ramDiskReplicaLru);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d01afb2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java
index 7507925..335ed70 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskReplicaTracker.java
@@ -45,6 +45,7 @@ public abstract class RamDiskReplicaTracker {
private final long blockId;
private File savedBlockFile;
private File savedMetaFile;
+ private long lockedBytesReserved;
private long creationTime;
protected AtomicLong numReads = new AtomicLong(0);
@@ -61,10 +62,12 @@ public abstract class RamDiskReplicaTracker {
FsVolumeImpl lazyPersistVolume;
RamDiskReplica(final String bpid, final long blockId,
- final FsVolumeImpl ramDiskVolume) {
+ final FsVolumeImpl ramDiskVolume,
+ long lockedBytesReserved) {
this.bpid = bpid;
this.blockId = blockId;
this.ramDiskVolume = ramDiskVolume;
+ this.lockedBytesReserved = lockedBytesReserved;
lazyPersistVolume = null;
savedMetaFile = null;
savedBlockFile = null;
@@ -168,6 +171,10 @@ public abstract class RamDiskReplicaTracker {
public String toString() {
return "[BlockPoolID=" + bpid + "; BlockId=" + blockId + "]";
}
+
+ public long getLockedBytesReserved() {
+ return lockedBytesReserved;
+ }
}
/**
@@ -201,7 +208,8 @@ public abstract class RamDiskReplicaTracker {
* @param transientVolume RAM disk volume that stores the replica.
*/
abstract void addReplica(final String bpid, final long blockId,
- final FsVolumeImpl transientVolume);
+ final FsVolumeImpl transientVolume,
+ long lockedBytesReserved);
/**
* Invoked when a replica is opened by a client. This may be used as
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d01afb2a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 12ad23e..fdbacdc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -1582,7 +1582,7 @@ public class MiniDFSCluster {
throw new IllegalStateException("Attempting to finalize "
+ "Namenode but it is not running");
}
- ToolRunner.run(new DFSAdmin(conf), new String[] {"-finalizeUpgrade"});
+ ToolRunner.run(new DFSAdmin(conf), new String[]{"-finalizeUpgrade"});
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d01afb2a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
index e756f0b..92d31d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.hdfs.server.balancer.Balancer.Parameters;
import org.apache.hadoop.hdfs.server.balancer.Balancer.Result;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.LazyPersistTestCase;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
@@ -120,13 +121,16 @@ public class TestBalancer {
conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L);
}
- static void initConfWithRamDisk(Configuration conf) {
+ static void initConfWithRamDisk(Configuration conf,
+ long ramDiskCapacity) {
conf.setLong(DFS_BLOCK_SIZE_KEY, DEFAULT_RAM_DISK_BLOCK_SIZE);
+ conf.setLong(DFS_DATANODE_MAX_LOCKED_MEMORY_KEY, ramDiskCapacity);
conf.setInt(DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC, 3);
conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, 1);
conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500);
conf.setInt(DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC, 1);
conf.setInt(DFS_DATANODE_RAM_DISK_LOW_WATERMARK_BYTES, DEFAULT_RAM_DISK_BLOCK_SIZE);
+ LazyPersistTestCase.initCacheManipulator();
}
/* create a file with a length of <code>fileLen</code> */
@@ -1245,7 +1249,6 @@ public class TestBalancer {
final int SEED = 0xFADED;
final short REPL_FACT = 1;
Configuration conf = new Configuration();
- initConfWithRamDisk(conf);
final int defaultRamDiskCapacity = 10;
final long ramDiskStorageLimit =
@@ -1255,6 +1258,8 @@ public class TestBalancer {
((long) defaultRamDiskCapacity * DEFAULT_RAM_DISK_BLOCK_SIZE) +
(DEFAULT_RAM_DISK_BLOCK_SIZE - 1);
+ initConfWithRamDisk(conf, ramDiskStorageLimit);
+
cluster = new MiniDFSCluster
.Builder(conf)
.numDataNodes(1)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d01afb2a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index 2ac9416..778dd28 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -492,6 +492,10 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
}
@Override
+ public void releaseLockedMemory(long bytesToRelease) {
+ }
+
+ @Override
public void releaseReservedSpace(long bytesToRelease) {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d01afb2a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
index b225e35..9b942b7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.LazyPersistTestCase;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Test;
@@ -79,6 +80,8 @@ public class TestDirectoryScanner {
CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_LENGTH);
CONF.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1);
CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
+ CONF.setLong(DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY,
+ Long.MAX_VALUE);
}
/** create a file with a length of <code>fileLen</code> */
@@ -308,6 +311,7 @@ public class TestDirectoryScanner {
@Test (timeout=300000)
public void testRetainBlockOnPersistentStorage() throws Exception {
+ LazyPersistTestCase.initCacheManipulator();
cluster = new MiniDFSCluster
.Builder(CONF)
.storageTypes(new StorageType[] { StorageType.RAM_DISK, StorageType.DEFAULT })
@@ -349,6 +353,7 @@ public class TestDirectoryScanner {
@Test (timeout=300000)
public void testDeleteBlockOnTransientStorage() throws Exception {
+ LazyPersistTestCase.initCacheManipulator();
cluster = new MiniDFSCluster
.Builder(CONF)
.storageTypes(new StorageType[] { StorageType.RAM_DISK, StorageType.DEFAULT })
@@ -615,6 +620,10 @@ public class TestDirectoryScanner {
}
@Override
+ public void releaseLockedMemory(long bytesToRelease) {
+ }
+
+ @Override
public BlockIterator newBlockIterator(String bpid, String name) {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d01afb2a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
index 7a09630..58932fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
@@ -339,7 +339,7 @@ public class TestFsDatasetCache {
for (int i=0; i<numFiles-1; i++) {
setHeartbeatResponse(cacheBlocks(fileLocs[i]));
total = DFSTestUtil.verifyExpectedCacheUsage(
- rounder.round(total + fileSizes[i]), 4 * (i + 1), fsd);
+ rounder.roundUp(total + fileSizes[i]), 4 * (i + 1), fsd);
}
// nth file should hit a capacity exception
@@ -365,7 +365,7 @@ public class TestFsDatasetCache {
int curCachedBlocks = 16;
for (int i=0; i<numFiles-1; i++) {
setHeartbeatResponse(uncacheBlocks(fileLocs[i]));
- long uncachedBytes = rounder.round(fileSizes[i]);
+ long uncachedBytes = rounder.roundUp(fileSizes[i]);
total -= uncachedBytes;
curCachedBlocks -= uncachedBytes / BLOCK_SIZE;
DFSTestUtil.verifyExpectedCacheUsage(total, curCachedBlocks, fsd);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d01afb2a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java
index ea9e4c1..3242ff7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalVolumeImpl.java
@@ -82,6 +82,10 @@ public class ExternalVolumeImpl implements FsVolumeSpi {
}
@Override
+ public void releaseLockedMemory(long bytesToRelease) {
+ }
+
+ @Override
public BlockIterator newBlockIterator(String bpid, String name) {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d01afb2a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
index 5dc86f7..5ce5cc6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
@@ -23,16 +23,7 @@ import static org.apache.hadoop.fs.CreateFlag.CREATE;
import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST;
import static org.apache.hadoop.fs.StorageType.DEFAULT;
import static org.apache.hadoop.fs.StorageType.RAM_DISK;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CONTEXT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_RAM_DISK_LOW_WATERMARK_BYTES;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
@@ -40,6 +31,7 @@ import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.HashSet;
@@ -68,6 +60,7 @@ import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.tools.JMXGet;
+import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.net.unix.TemporarySocketDirectory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
@@ -80,8 +73,8 @@ public abstract class LazyPersistTestCase {
static final byte LAZY_PERSIST_POLICY_ID = (byte) 15;
static {
- DFSTestUtil.setNameNodeLogLevel(Level.ALL);
- GenericTestUtils.setLogLevel(FsDatasetImpl.LOG, Level.ALL);
+ DFSTestUtil.setNameNodeLogLevel(Level.DEBUG);
+ GenericTestUtils.setLogLevel(FsDatasetImpl.LOG, Level.DEBUG);
}
protected static final int BLOCK_SIZE = 5 * 1024 * 1024;
@@ -95,6 +88,8 @@ public abstract class LazyPersistTestCase {
protected static final int LAZY_WRITER_INTERVAL_SEC = 1;
protected static final Log LOG = LogFactory.getLog(LazyPersistTestCase.class);
protected static final short REPL_FACTOR = 1;
+ protected final long osPageSize =
+ NativeIO.POSIX.getCacheManipulator().getOperatingSystemPageSize();
protected MiniDFSCluster cluster;
protected DistributedFileSystem fs;
@@ -194,7 +189,7 @@ public abstract class LazyPersistTestCase {
protected final void makeRandomTestFile(Path path, long length,
boolean isLazyPersist, long seed) throws IOException {
DFSTestUtil.createFile(fs, path, isLazyPersist, BUFFER_LENGTH, length,
- BLOCK_SIZE, REPL_FACTOR, seed, true);
+ BLOCK_SIZE, REPL_FACTOR, seed, true);
}
protected final void makeTestFile(Path path, long length,
@@ -242,10 +237,12 @@ public abstract class LazyPersistTestCase {
int ramDiskReplicaCapacity,
long ramDiskStorageLimit,
long evictionLowWatermarkReplicas,
+ long maxLockedMemory,
boolean useSCR,
boolean useLegacyBlockReaderLocal,
boolean disableScrubber) throws IOException {
+ initCacheManipulator();
Configuration conf = new Configuration();
conf.setLong(DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
if (disableScrubber) {
@@ -262,6 +259,7 @@ public abstract class LazyPersistTestCase {
conf.setLong(DFS_DATANODE_RAM_DISK_LOW_WATERMARK_BYTES,
evictionLowWatermarkReplicas * BLOCK_SIZE);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY, 1);
+ conf.setLong(DFS_DATANODE_MAX_LOCKED_MEMORY_KEY, maxLockedMemory);
if (useSCR) {
conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, true);
@@ -311,6 +309,31 @@ public abstract class LazyPersistTestCase {
LOG.info("Cluster startup complete");
}
+ /**
+ * Use a dummy cache manipulator for testing.
+ */
+ public static void initCacheManipulator() {
+ NativeIO.POSIX.setCacheManipulator(new NativeIO.POSIX.CacheManipulator() {
+ @Override
+ public void mlock(String identifier,
+ ByteBuffer mmap, long length) throws IOException {
+ LOG.info("LazyPersistTestCase: faking mlock of " + identifier + " bytes.");
+ }
+
+ @Override
+ public long getMemlockLimit() {
+ LOG.info("LazyPersistTestCase: fake return " + Long.MAX_VALUE);
+ return Long.MAX_VALUE;
+ }
+
+ @Override
+ public boolean verifyCanMlock() {
+ LOG.info("LazyPersistTestCase: fake return " + true);
+ return true;
+ }
+ });
+ }
+
ClusterWithRamDiskBuilder getClusterBuilder() {
return new ClusterWithRamDiskBuilder();
}
@@ -344,6 +367,11 @@ public abstract class LazyPersistTestCase {
return this;
}
+ public ClusterWithRamDiskBuilder setMaxLockedMemory(long maxLockedMemory) {
+ this.maxLockedMemory = maxLockedMemory;
+ return this;
+ }
+
public ClusterWithRamDiskBuilder setUseScr(boolean useScr) {
this.useScr = useScr;
return this;
@@ -376,13 +404,14 @@ public abstract class LazyPersistTestCase {
LazyPersistTestCase.this.startUpCluster(
numDatanodes, hasTransientStorage, storageTypes, ramDiskReplicaCapacity,
ramDiskStorageLimit, evictionLowWatermarkReplicas,
- useScr, useLegacyBlockReaderLocal,disableScrubber);
+ maxLockedMemory, useScr, useLegacyBlockReaderLocal, disableScrubber);
}
private int numDatanodes = REPL_FACTOR;
private StorageType[] storageTypes = null;
private int ramDiskReplicaCapacity = -1;
private long ramDiskStorageLimit = -1;
+ private long maxLockedMemory = Long.MAX_VALUE;
private boolean hasTransientStorage = true;
private boolean useScr = false;
private boolean useLegacyBlockReaderLocal = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d01afb2a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistLockedMemory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistLockedMemory.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistLockedMemory.java
new file mode 100644
index 0000000..9ea4665
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistLockedMemory.java
@@ -0,0 +1,201 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
+
+
+import com.google.common.base.Supplier;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSOutputStream;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.MetricsAsserts;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.hadoop.fs.CreateFlag.CREATE;
+import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST;
+import static org.apache.hadoop.fs.StorageType.DEFAULT;
+import static org.apache.hadoop.fs.StorageType.RAM_DISK;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * Verify that locked memory is used correctly when writing to replicas in
+ * memory
+ */
+public class TestLazyPersistLockedMemory extends LazyPersistTestCase {
+
+ /**
+ * RAM disk present but locked memory is set to zero. Placement should
+ * fall back to disk.
+ */
+ @Test
+ public void testWithNoLockedMemory() throws IOException {
+ getClusterBuilder().setNumDatanodes(1)
+ .setMaxLockedMemory(0).build();
+
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+ makeTestFile(path, BLOCK_SIZE, true);
+ ensureFileReplicasOnStorageType(path, DEFAULT);
+ }
+
+ @Test
+ public void testReservation()
+ throws IOException, TimeoutException, InterruptedException {
+ getClusterBuilder().setNumDatanodes(1)
+ .setMaxLockedMemory(BLOCK_SIZE).build();
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ final FsDatasetSpi<?> fsd = cluster.getDataNodes().get(0).getFSDataset();
+
+ // Create a file and ensure the replica in RAM_DISK uses locked memory.
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+ makeTestFile(path, BLOCK_SIZE, true);
+ ensureFileReplicasOnStorageType(path, RAM_DISK);
+ assertThat(fsd.getCacheUsed(), is((long) BLOCK_SIZE));
+ }
+
+ @Test
+ public void testReleaseOnFileDeletion()
+ throws IOException, TimeoutException, InterruptedException {
+ getClusterBuilder().setNumDatanodes(1)
+ .setMaxLockedMemory(BLOCK_SIZE).build();
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ final FsDatasetSpi<?> fsd = cluster.getDataNodes().get(0).getFSDataset();
+
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+ makeTestFile(path, BLOCK_SIZE, true);
+ ensureFileReplicasOnStorageType(path, RAM_DISK);
+ assertThat(fsd.getCacheUsed(), is((long) BLOCK_SIZE));
+
+ // Delete the file and ensure that the locked memory is released.
+ fs.delete(path, false);
+ DataNodeTestUtils.triggerBlockReport(cluster.getDataNodes().get(0));
+ waitForLockedBytesUsed(fsd, 0);
+ }
+
+ /**
+ * Verify that locked RAM is released when blocks are evicted from RAM disk.
+ */
+ @Test
+ public void testReleaseOnEviction()
+ throws IOException, TimeoutException, InterruptedException {
+ getClusterBuilder().setNumDatanodes(1)
+ .setMaxLockedMemory(BLOCK_SIZE)
+ .setRamDiskReplicaCapacity(BLOCK_SIZE * 2 - 1)
+ .build();
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ final FsDatasetSpi<?> fsd = cluster.getDataNodes().get(0).getFSDataset();
+
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+ makeTestFile(path, BLOCK_SIZE, true);
+
+ // The block should get evicted soon since it pushes RAM disk free
+ // space below the threshold.
+ waitForLockedBytesUsed(fsd, 0);
+
+ MetricsRecordBuilder rb =
+ MetricsAsserts.getMetrics(cluster.getDataNodes().get(0).getMetrics().name());
+ MetricsAsserts.assertCounter("RamDiskBlocksEvicted", 1L, rb);
+ }
+
+ /**
+ * Verify that locked bytes are correctly updated when a block is finalized
+ * at less than its max length.
+ */
+ @Test
+ public void testShortBlockFinalized()
+ throws IOException, TimeoutException, InterruptedException {
+ getClusterBuilder().setNumDatanodes(1).build();
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ final FsDatasetSpi<?> fsd = cluster.getDataNodes().get(0).getFSDataset();
+
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+ makeTestFile(path, 1, true);
+ assertThat(fsd.getCacheUsed(), is(osPageSize));
+
+ // Delete the file and ensure locked RAM usage goes to zero.
+ fs.delete(path, false);
+ waitForLockedBytesUsed(fsd, 0);
+ }
+
+ /**
+ * Verify that locked bytes are correctly updated when the client goes
+ * away unexpectedly during a write.
+ */
+ @Test
+ public void testWritePipelineFailure()
+ throws IOException, TimeoutException, InterruptedException {
+ getClusterBuilder().setNumDatanodes(1).build();
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ final FsDatasetSpi<?> fsd = cluster.getDataNodes().get(0).getFSDataset();
+
+ Path path = new Path("/" + METHOD_NAME + ".dat");
+
+ EnumSet<CreateFlag> createFlags = EnumSet.of(CREATE, LAZY_PERSIST);
+ // Write 1 byte to the file and kill the writer.
+ final FSDataOutputStream fos =
+ fs.create(path,
+ FsPermission.getFileDefault(),
+ createFlags,
+ BUFFER_LENGTH,
+ REPL_FACTOR,
+ BLOCK_SIZE,
+ null);
+
+ fos.write(new byte[1]);
+ fos.hsync();
+ DFSTestUtil.abortStream((DFSOutputStream) fos.getWrappedStream());
+ waitForLockedBytesUsed(fsd, osPageSize);
+
+ // Delete the file and ensure locked RAM goes to zero.
+ fs.delete(path, false);
+ DataNodeTestUtils.triggerBlockReport(cluster.getDataNodes().get(0));
+ waitForLockedBytesUsed(fsd, 0);
+ }
+
+ /**
+ * Wait until used locked byte count goes to the expected value.
+ * @throws TimeoutException after 300 seconds.
+ */
+ private void waitForLockedBytesUsed(final FsDatasetSpi<?> fsd,
+ final long expectedLockedBytes)
+ throws TimeoutException, InterruptedException {
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ long cacheUsed = fsd.getCacheUsed();
+ LOG.info("cacheUsed=" + cacheUsed + ", waiting for it to be " + expectedLockedBytes);
+ if (cacheUsed < 0) {
+ throw new IllegalStateException("cacheUsed unpexpectedly negative");
+ }
+ return (cacheUsed == expectedLockedBytes);
+ }
+ }, 1000, 300000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d01afb2a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
index d5664cf..a77184b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
@@ -204,7 +204,7 @@ public class TestWriteToReplica {
long available = v.getCapacity()-v.getDfsUsed();
long expectedLen = blocks[FINALIZED].getNumBytes();
try {
- v.decDfsUsed(bpid, -available);
+ v.onBlockFileDeletion(bpid, -available);
blocks[FINALIZED].setNumBytes(expectedLen+100);
dataSet.append(blocks[FINALIZED], newGS, expectedLen);
Assert.fail("Should not have space to append to an RWR replica" + blocks[RWR]);
@@ -212,7 +212,7 @@ public class TestWriteToReplica {
Assert.assertTrue(e.getMessage().startsWith(
"Insufficient space for appending to "));
}
- v.decDfsUsed(bpid, available);
+ v.onBlockFileDeletion(bpid, available);
blocks[FINALIZED].setNumBytes(expectedLen);
newGS = blocks[RBW].getGenerationStamp()+1;
[02/50] [abbrv] hadoop git commit: YARN-3526. ApplicationMaster
tracking URL is incorrectly redirected on a QJM cluster. Contributed by
Weiwei Yang
Posted by zj...@apache.org.
YARN-3526. ApplicationMaster tracking URL is incorrectly redirected on a QJM cluster. Contributed by Weiwei Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7f53ed0c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7f53ed0c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7f53ed0c
Branch: refs/heads/YARN-2928
Commit: 7f53ed0c1194998595a945bdf7d2c2137b4f475e
Parents: cf050a3
Author: Xuan <xg...@apache.org>
Authored: Fri May 15 22:39:49 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:47:00 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../test/java/org/apache/hadoop/yarn/client/TestRMFailover.java | 3 +++
.../hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java | 2 ++
3 files changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f53ed0c/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5284f9c..074aef0 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -584,6 +584,9 @@ Release 2.7.1 - UNRELEASED
YARN-3641. NodeManager: stopRecoveryStore() shouldn't be skipped when
exceptions happen in stopping NM's sub-services. (Junping Du via jlowe)
+ YARN-3526. ApplicationMaster tracking URL is incorrectly redirected
+ on a QJM cluster. (Weiwei Yang via xgong)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f53ed0c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
index 4938255..d4fc5c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
@@ -326,6 +326,9 @@ public class TestRMFailover extends ClientBaseWithFixes {
header = getHeader("Refresh", rm2Url + "/ws/v1/cluster/apps");
assertTrue(header.contains("; url=" + rm1Url));
+ header = getHeader("Refresh", rm2Url + "/proxy/" + fakeAppId);
+ assertEquals(null, header);
+
// Due to the limitation of MiniYARNCluster and dispatcher is a singleton,
// we couldn't add the test case after explicitFailover();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f53ed0c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
index b1027a8..500f17a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
@@ -30,6 +30,7 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.hadoop.http.HtmlQuoting;
+import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import com.google.common.collect.Sets;
import com.google.inject.Injector;
@@ -88,6 +89,7 @@ public class RMWebAppFilter extends GuiceContainer {
private boolean shouldRedirect(RMWebApp rmWebApp, String uri) {
return !uri.equals("/" + rmWebApp.wsName() + "/v1/cluster/info")
&& !uri.equals("/" + rmWebApp.name() + "/cluster")
+ && !uri.startsWith(ProxyUriUtils.PROXY_BASE)
&& !NON_REDIRECTED_URIS.contains(uri);
}
}
[26/50] [abbrv] hadoop git commit: HDFS-8131. Implement a space
balanced block placement policy. Contributed by Liu Shaohui.
Posted by zj...@apache.org.
HDFS-8131. Implement a space balanced block placement policy. Contributed by Liu Shaohui.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8812d832
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8812d832
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8812d832
Branch: refs/heads/YARN-2928
Commit: 8812d8321d4991e4c91b63c2ccaf02b6ad12f62c
Parents: 84cd964
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue May 19 08:04:38 2015 -0500
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:52:09 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 +-
.../AvailableSpaceBlockPlacementPolicy.java | 95 +++++++++++
.../BlockPlacementPolicyDefault.java | 11 +-
.../TestAvailableSpaceBlockPlacementPolicy.java | 167 +++++++++++++++++++
5 files changed, 279 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8812d832/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e5fcba2..76888a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -345,6 +345,9 @@ Release 2.8.0 - UNRELEASED
HDFS-7891. A block placement policy with best rack failure tolerance.
(Walter Su via szetszwo)
+ HDFS-8131. Implement a space balanced block placement policy (Liu Shaohui
+ via kihwal)
+
IMPROVEMENTS
HDFS-3918. EditLogTailer shouldn't log WARN when other node
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8812d832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index c903e76..9c19f91 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -519,7 +519,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY = "dfs.web.authentication.kerberos.keytab";
public static final String DFS_NAMENODE_MAX_OP_SIZE_KEY = "dfs.namenode.max.op.size";
public static final int DFS_NAMENODE_MAX_OP_SIZE_DEFAULT = 50 * 1024 * 1024;
-
+ public static final String DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY =
+ "dfs.namenode.available-space-block-placement-policy.balanced-space-preference-fraction";
+ public static final float DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT =
+ 0.6f;
+
public static final String DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY = "dfs.block.local-path-access.user";
public static final String DFS_DOMAIN_SOCKET_PATH_KEY = "dfs.domain.socket.path";
public static final String DFS_DOMAIN_SOCKET_PATH_DEFAULT = "";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8812d832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
new file mode 100644
index 0000000..74c1c78
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT;
+
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.net.NetworkTopology;
+
+/**
+ * Space balanced block placement policy.
+ */
+public class AvailableSpaceBlockPlacementPolicy extends
+ BlockPlacementPolicyDefault {
+ private static final Log LOG = LogFactory
+ .getLog(AvailableSpaceBlockPlacementPolicy.class);
+ private static final Random RAND = new Random();
+ private int balancedPreference =
+ (int) (100 * DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT);
+
+ @Override
+ public void initialize(Configuration conf, FSClusterStats stats,
+ NetworkTopology clusterMap, Host2NodesMap host2datanodeMap) {
+ super.initialize(conf, stats, clusterMap, host2datanodeMap);
+ float balancedPreferencePercent =
+ conf.getFloat(
+ DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY,
+ DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT);
+
+ LOG.info("Available space block placement policy initialized: "
+ + DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY
+ + " = " + balancedPreferencePercent);
+
+ if (balancedPreferencePercent > 1.0) {
+ LOG.warn("The value of "
+ + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY
+ + " is greater than 1.0 but should be in the range 0.0 - 1.0");
+ }
+ if (balancedPreferencePercent < 0.5) {
+ LOG.warn("The value of "
+ + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY
+ + " is less than 0.5 so datanodes with more used percent will"
+ + " receive more block allocations.");
+ }
+ balancedPreference = (int) (100 * balancedPreferencePercent);
+ }
+
+ @Override
+ protected DatanodeDescriptor chooseDataNode(String scope) {
+ DatanodeDescriptor a = (DatanodeDescriptor) clusterMap.chooseRandom(scope);
+ DatanodeDescriptor b = (DatanodeDescriptor) clusterMap.chooseRandom(scope);
+ int ret = compareDataNode(a, b);
+ if (ret == 0) {
+ return a;
+ } else if (ret < 0) {
+ return (RAND.nextInt(100) < balancedPreference) ? a : b;
+ } else {
+ return (RAND.nextInt(100) < balancedPreference) ? b : a;
+ }
+ }
+
+ /**
+ * Compare the two data nodes.
+ */
+ protected int compareDataNode(final DatanodeDescriptor a,
+ final DatanodeDescriptor b) {
+ if (a.equals(b)
+ || Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) < 5) {
+ return 0;
+ }
+ return a.getDfsUsedPercent() < b.getDfsUsedPercent() ? -1 : 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8812d832/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index 620d2a6..21ad01d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -646,8 +646,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
boolean badTarget = false;
DatanodeStorageInfo firstChosen = null;
while(numOfReplicas > 0 && numOfAvailableNodes > 0) {
- DatanodeDescriptor chosenNode =
- (DatanodeDescriptor)clusterMap.chooseRandom(scope);
+ DatanodeDescriptor chosenNode = chooseDataNode(scope);
if (excludedNodes.add(chosenNode)) { //was not in the excluded list
if (LOG.isDebugEnabled()) {
builder.append("\nNode ").append(NodeBase.getPath(chosenNode)).append(" [");
@@ -709,6 +708,14 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
}
/**
+ * Choose a datanode from the given <i>scope</i>.
+ * @return the chosen node, if there is any.
+ */
+ protected DatanodeDescriptor chooseDataNode(final String scope) {
+ return (DatanodeDescriptor) clusterMap.chooseRandom(scope);
+ }
+
+ /**
* If the given storage is a good target, add it to the result list and
* update the set of excluded nodes.
* @return -1 if the given is not a good target;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8812d832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java
new file mode 100644
index 0000000..f1e4e1c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import java.io.File;
+import java.util.ArrayList;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.TestBlockStoragePolicy;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.test.PathUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestAvailableSpaceBlockPlacementPolicy {
+ private final static int numRacks = 4;
+ private final static int nodesPerRack = 5;
+ private final static int blockSize = 1024;
+ private final static int chooseTimes = 10000;
+ private final static String file = "/tobers/test";
+ private final static int replica = 3;
+
+ private static DatanodeStorageInfo[] storages;
+ private static DatanodeDescriptor[] dataNodes;
+ private static Configuration conf;
+ private static NameNode namenode;
+ private static BlockPlacementPolicy placementPolicy;
+ private static NetworkTopology cluster;
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ conf = new HdfsConfiguration();
+ conf.setFloat(
+ DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY,
+ 0.6f);
+ String[] racks = new String[numRacks];
+ for (int i = 0; i < numRacks; i++) {
+ racks[i] = "/rack" + i;
+ }
+
+ String[] owerRackOfNodes = new String[numRacks * nodesPerRack];
+ for (int i = 0; i < nodesPerRack; i++) {
+ for (int j = 0; j < numRacks; j++) {
+ owerRackOfNodes[i * numRacks + j] = racks[j];
+ }
+ }
+
+ storages = DFSTestUtil.createDatanodeStorageInfos(owerRackOfNodes);
+ dataNodes = DFSTestUtil.toDatanodeDescriptor(storages);
+
+ FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
+ conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
+ File baseDir = PathUtils.getTestDir(AvailableSpaceBlockPlacementPolicy.class);
+ conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, new File(baseDir, "name").getPath());
+ conf.set(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
+ AvailableSpaceBlockPlacementPolicy.class.getName());
+
+ DFSTestUtil.formatNameNode(conf);
+ namenode = new NameNode(conf);
+
+ final BlockManager bm = namenode.getNamesystem().getBlockManager();
+ placementPolicy = bm.getBlockPlacementPolicy();
+ cluster = bm.getDatanodeManager().getNetworkTopology();
+ for (int i = 0; i < nodesPerRack * numRacks; i++) {
+ cluster.add(dataNodes[i]);
+ }
+
+ setupDataNodeCapacity();
+ }
+
+ private static void updateHeartbeatWithUsage(DatanodeDescriptor dn,
+ long capacity, long dfsUsed, long remaining, long blockPoolUsed,
+ long dnCacheCapacity, long dnCacheUsed, int xceiverCount,
+ int volFailures) {
+ dn.getStorageInfos()[0].setUtilizationForTesting(
+ capacity, dfsUsed, remaining, blockPoolUsed);
+ dn.updateHeartbeat(
+ BlockManagerTestUtil.getStorageReportsForDatanode(dn),
+ dnCacheCapacity, dnCacheUsed, xceiverCount, volFailures, null);
+ }
+
+ private static void setupDataNodeCapacity() {
+ for (int i = 0; i < nodesPerRack * numRacks; i++) {
+ if ((i % 2) == 0) {
+ // remaining 100%
+ updateHeartbeatWithUsage(dataNodes[i], 2 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize,
+ 0L, 2 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, 0L, 0L, 0L, 0, 0);
+ } else {
+ // remaining 50%
+ updateHeartbeatWithUsage(dataNodes[i], 2 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize,
+ HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, HdfsServerConstants.MIN_BLOCKS_FOR_WRITE
+ * blockSize, 0L, 0L, 0L, 0, 0);
+ }
+ }
+ }
+
+ /*
+ * To verify that the BlockPlacementPolicy can be replaced by AvailableSpaceBlockPlacementPolicy via
+ * changing the configuration.
+ */
+ @Test
+ public void testPolicyReplacement() {
+ Assert.assertTrue((placementPolicy instanceof AvailableSpaceBlockPlacementPolicy));
+ }
+
+ /*
+ * Call choose target many times and verify that nodes with more remaining percent will be chosen
+ * with high possibility.
+ */
+ @Test
+ public void testChooseTarget() {
+ int total = 0;
+ int moreRemainingNode = 0;
+ for (int i = 0; i < chooseTimes; i++) {
+ DatanodeStorageInfo[] targets =
+ namenode
+ .getNamesystem()
+ .getBlockManager()
+ .getBlockPlacementPolicy()
+ .chooseTarget(file, replica, null, new ArrayList<DatanodeStorageInfo>(), false, null,
+ blockSize, TestBlockStoragePolicy.DEFAULT_STORAGE_POLICY);
+
+ Assert.assertTrue(targets.length == replica);
+ for (int j = 0; j < replica; j++) {
+ total++;
+ if (targets[j].getDatanodeDescriptor().getRemainingPercent() > 60) {
+ moreRemainingNode++;
+ }
+ }
+ }
+ Assert.assertTrue(total == replica * chooseTimes);
+ double possibility = 1.0 * moreRemainingNode / total;
+ Assert.assertTrue(possibility > 0.52);
+ Assert.assertTrue(possibility < 0.55);
+ }
+
+ @AfterClass
+ public static void teardownCluster() {
+ if (namenode != null) {
+ namenode.stop();
+ }
+ }
+}
[23/50] [abbrv] hadoop git commit: Move HADOOP-8934 in CHANGES.txt
from 3.0.0 to 2.8.0.
Posted by zj...@apache.org.
Move HADOOP-8934 in CHANGES.txt from 3.0.0 to 2.8.0.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ab23610
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ab23610
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ab23610
Branch: refs/heads/YARN-2928
Commit: 5ab23610332aeb1c58c274b22d578c6473468a45
Parents: d48e623
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue May 19 18:01:24 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:52:08 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ab23610/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 8ce77b6..ee7d1e3 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -38,9 +38,6 @@ Trunk (Unreleased)
HADOOP-11485. Pluggable shell integration (aw)
- HADOOP-8934. Shell command ls should include sort options (Jonathan Allen
- via aw)
-
HADOOP-11554. Expose HadoopKerberosName as a hadoop subcommand (aw)
HADOOP-11565. Add --slaves shell option (aw)
@@ -488,6 +485,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11949. Add user-provided plugins to test-patch (Sean Busbey via aw)
+ HADOOP-8934. Shell command ls should include sort options (Jonathan Allen
+ via aw)
+
HADOOP-10971. Add -C flag to make `hadoop fs -ls` print filenames only.
(Kengo Seki via aajisaka)
[41/50] [abbrv] hadoop git commit: HADOOP-11698. Remove DistCpV1 and
Logalyzer. Contributed by Brahma Reddy Battula.
Posted by zj...@apache.org.
HADOOP-11698. Remove DistCpV1 and Logalyzer. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/850cc678
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/850cc678
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/850cc678
Branch: refs/heads/YARN-2928
Commit: 850cc678c8ebd266d9e88df92bca4fdc55c0205b
Parents: 440ba1e
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed May 20 19:25:45 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 12:42:56 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../java/org/apache/hadoop/tools/DistCpV1.java | 1674 ------------------
.../java/org/apache/hadoop/tools/Logalyzer.java | 329 ----
.../org/apache/hadoop/tools/TestCopyFiles.java | 1077 -----------
.../org/apache/hadoop/tools/TestLogalyzer.java | 133 --
5 files changed, 3 insertions(+), 3213 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/850cc678/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index b0b8fb7..1624ce2 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -26,6 +26,9 @@ Trunk (Unreleased)
HADOOP-11627. Remove io.native.lib.available.
(Brahma Reddy Battula via aajisaka)
+ HADOOP-11698. Remove DistCpV1 and Logalyzer.
+ (Brahma Reddy Battula via aajisaka)
+
NEW FEATURES
HADOOP-6590. Add a username check for hadoop sub-commands (John Smith via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/850cc678/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCpV1.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCpV1.java b/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCpV1.java
deleted file mode 100644
index 39ac5c3..0000000
--- a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCpV1.java
+++ /dev/null
@@ -1,1674 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.tools;
-
-import java.io.BufferedReader;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Random;
-import java.util.Stack;
-import java.util.StringTokenizer;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileChecksum;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.Trash;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.Reader;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.io.SequenceFile.Writer;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.FileSplit;
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.InvalidInputException;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.RecordReader;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.SequenceFileRecordReader;
-import org.apache.hadoop.mapreduce.JobSubmissionFiles;
-import org.apache.hadoop.mapreduce.security.TokenCache;
-import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
-
-/**
- * A Map-reduce program to recursively copy directories between
- * different file-systems.
- */
-@Deprecated
-public class DistCpV1 implements Tool {
- public static final Log LOG = LogFactory.getLog(DistCpV1.class);
-
- private static final String NAME = "distcp";
-
- private static final String usage = NAME
- + " [OPTIONS] <srcurl>* <desturl>" +
- "\n\nOPTIONS:" +
- "\n-p[rbugpt] Preserve status" +
- "\n r: replication number" +
- "\n b: block size" +
- "\n u: user" +
- "\n g: group" +
- "\n p: permission" +
- "\n t: modification and access times" +
- "\n -p alone is equivalent to -prbugpt" +
- "\n-i Ignore failures" +
- "\n-basedir <basedir> Use <basedir> as the base directory when copying files from <srcurl>" +
- "\n-log <logdir> Write logs to <logdir>" +
- "\n-m <num_maps> Maximum number of simultaneous copies" +
- "\n-overwrite Overwrite destination" +
- "\n-update Overwrite if src size different from dst size" +
- "\n-skipcrccheck Do not use CRC check to determine if src is " +
- "\n different from dest. Relevant only if -update" +
- "\n is specified" +
- "\n-f <urilist_uri> Use list at <urilist_uri> as src list" +
- "\n-filelimit <n> Limit the total number of files to be <= n" +
- "\n-sizelimit <n> Limit the total size to be <= n bytes" +
- "\n-delete Delete the files existing in the dst but not in src" +
- "\n-dryrun Display count of files and total size of files" +
- "\n in src and then exit. Copy is not done at all." +
- "\n desturl should not be speicified with out -update." +
- "\n-mapredSslConf <f> Filename of SSL configuration for mapper task" +
-
- "\n\nNOTE 1: if -overwrite or -update are set, each source URI is " +
- "\n interpreted as an isomorphic update to an existing directory." +
- "\nFor example:" +
- "\nhadoop " + NAME + " -p -update \"hdfs://A:8020/user/foo/bar\" " +
- "\"hdfs://B:8020/user/foo/baz\"\n" +
- "\n would update all descendants of 'baz' also in 'bar'; it would " +
- "\n *not* update /user/foo/baz/bar" +
-
- "\n\nNOTE 2: The parameter <n> in -filelimit and -sizelimit can be " +
- "\n specified with symbolic representation. For examples," +
- "\n 1230k = 1230 * 1024 = 1259520" +
- "\n 891g = 891 * 1024^3 = 956703965184" +
-
- "\n";
-
- private static final long BYTES_PER_MAP = 256 * 1024 * 1024;
- private static final int MAX_MAPS_PER_NODE = 20;
- private static final int SYNC_FILE_MAX = 10;
- private static final int DEFAULT_FILE_RETRIES = 3;
-
- static enum Counter { COPY, SKIP, FAIL, BYTESCOPIED, BYTESEXPECTED }
- static enum Options {
- DELETE("-delete", NAME + ".delete"),
- FILE_LIMIT("-filelimit", NAME + ".limit.file"),
- SIZE_LIMIT("-sizelimit", NAME + ".limit.size"),
- IGNORE_READ_FAILURES("-i", NAME + ".ignore.read.failures"),
- PRESERVE_STATUS("-p", NAME + ".preserve.status"),
- OVERWRITE("-overwrite", NAME + ".overwrite.always"),
- UPDATE("-update", NAME + ".overwrite.ifnewer"),
- SKIPCRC("-skipcrccheck", NAME + ".skip.crc.check");
-
- final String cmd, propertyname;
-
- private Options(String cmd, String propertyname) {
- this.cmd = cmd;
- this.propertyname = propertyname;
- }
-
- private long parseLong(String[] args, int offset) {
- if (offset == args.length) {
- throw new IllegalArgumentException("<n> not specified in " + cmd);
- }
- long n = StringUtils.TraditionalBinaryPrefix.string2long(args[offset]);
- if (n <= 0) {
- throw new IllegalArgumentException("n = " + n + " <= 0 in " + cmd);
- }
- return n;
- }
- }
- static enum FileAttribute {
- BLOCK_SIZE, REPLICATION, USER, GROUP, PERMISSION, TIMES;
-
- final char symbol;
-
- private FileAttribute() {
- symbol = StringUtils.toLowerCase(toString()).charAt(0);
- }
-
- static EnumSet<FileAttribute> parse(String s) {
- if (s == null || s.length() == 0) {
- return EnumSet.allOf(FileAttribute.class);
- }
-
- EnumSet<FileAttribute> set = EnumSet.noneOf(FileAttribute.class);
- FileAttribute[] attributes = values();
- for(char c : s.toCharArray()) {
- int i = 0;
- for(; i < attributes.length && c != attributes[i].symbol; i++);
- if (i < attributes.length) {
- if (!set.contains(attributes[i])) {
- set.add(attributes[i]);
- } else {
- throw new IllegalArgumentException("There are more than one '"
- + attributes[i].symbol + "' in " + s);
- }
- } else {
- throw new IllegalArgumentException("'" + c + "' in " + s
- + " is undefined.");
- }
- }
- return set;
- }
- }
-
- static final String TMP_DIR_LABEL = NAME + ".tmp.dir";
- static final String DST_DIR_LABEL = NAME + ".dest.path";
- static final String JOB_DIR_LABEL = NAME + ".job.dir";
- static final String MAX_MAPS_LABEL = NAME + ".max.map.tasks";
- static final String SRC_LIST_LABEL = NAME + ".src.list";
- static final String SRC_COUNT_LABEL = NAME + ".src.count";
- static final String TOTAL_SIZE_LABEL = NAME + ".total.size";
- static final String DST_DIR_LIST_LABEL = NAME + ".dst.dir.list";
- static final String BYTES_PER_MAP_LABEL = NAME + ".bytes.per.map";
- static final String PRESERVE_STATUS_LABEL
- = Options.PRESERVE_STATUS.propertyname + ".value";
- static final String FILE_RETRIES_LABEL = NAME + ".file.retries";
-
- private JobConf conf;
-
- public void setConf(Configuration conf) {
- if (conf instanceof JobConf) {
- this.conf = (JobConf) conf;
- } else {
- this.conf = new JobConf(conf);
- }
- }
-
- public Configuration getConf() {
- return conf;
- }
-
- public DistCpV1(Configuration conf) {
- setConf(conf);
- }
-
- /**
- * An input/output pair of filenames.
- */
- static class FilePair implements Writable {
- FileStatus input = new FileStatus();
- String output;
- FilePair() { }
- FilePair(FileStatus input, String output) {
- this.input = input;
- this.output = output;
- }
- public void readFields(DataInput in) throws IOException {
- input.readFields(in);
- output = Text.readString(in);
- }
- public void write(DataOutput out) throws IOException {
- input.write(out);
- Text.writeString(out, output);
- }
- public String toString() {
- return input + " : " + output;
- }
- }
-
- /**
- * InputFormat of a distcp job responsible for generating splits of the src
- * file list.
- */
- static class CopyInputFormat implements InputFormat<Text, Text> {
-
- /**
- * Produce splits such that each is no greater than the quotient of the
- * total size and the number of splits requested.
- * @param job The handle to the JobConf object
- * @param numSplits Number of splits requested
- */
- public InputSplit[] getSplits(JobConf job, int numSplits)
- throws IOException {
- int cnfiles = job.getInt(SRC_COUNT_LABEL, -1);
- long cbsize = job.getLong(TOTAL_SIZE_LABEL, -1);
- String srcfilelist = job.get(SRC_LIST_LABEL, "");
- if (cnfiles < 0 || cbsize < 0 || "".equals(srcfilelist)) {
- throw new RuntimeException("Invalid metadata: #files(" + cnfiles +
- ") total_size(" + cbsize + ") listuri(" +
- srcfilelist + ")");
- }
- Path src = new Path(srcfilelist);
- FileSystem fs = src.getFileSystem(job);
- FileStatus srcst = fs.getFileStatus(src);
-
- ArrayList<FileSplit> splits = new ArrayList<FileSplit>(numSplits);
- LongWritable key = new LongWritable();
- FilePair value = new FilePair();
- final long targetsize = cbsize / numSplits;
- long pos = 0L;
- long last = 0L;
- long acc = 0L;
- long cbrem = srcst.getLen();
- try (SequenceFile.Reader sl =
- new SequenceFile.Reader(job, Reader.file(src))) {
- for (; sl.next(key, value); last = sl.getPosition()) {
- // if adding this split would put this split past the target size,
- // cut the last split and put this next file in the next split.
- if (acc + key.get() > targetsize && acc != 0) {
- long splitsize = last - pos;
- splits.add(new FileSplit(src, pos, splitsize, (String[])null));
- cbrem -= splitsize;
- pos = last;
- acc = 0L;
- }
- acc += key.get();
- }
- }
- if (cbrem != 0) {
- splits.add(new FileSplit(src, pos, cbrem, (String[])null));
- }
-
- return splits.toArray(new FileSplit[splits.size()]);
- }
-
- /**
- * Returns a reader for this split of the src file list.
- */
- public RecordReader<Text, Text> getRecordReader(InputSplit split,
- JobConf job, Reporter reporter) throws IOException {
- return new SequenceFileRecordReader<Text, Text>(job, (FileSplit)split);
- }
- }
-
- /**
- * FSCopyFilesMapper: The mapper for copying files between FileSystems.
- */
- static class CopyFilesMapper
- implements Mapper<LongWritable, FilePair, WritableComparable<?>, Text> {
- // config
- private int sizeBuf = 128 * 1024;
- private FileSystem destFileSys = null;
- private boolean ignoreReadFailures;
- private boolean preserve_status;
- private EnumSet<FileAttribute> preseved;
- private boolean overwrite;
- private boolean update;
- private Path destPath = null;
- private byte[] buffer = null;
- private JobConf job;
- private boolean skipCRCCheck = false;
-
- // stats
- private int failcount = 0;
- private int skipcount = 0;
- private int copycount = 0;
-
- private String getCountString() {
- return "Copied: " + copycount + " Skipped: " + skipcount
- + " Failed: " + failcount;
- }
- private void updateStatus(Reporter reporter) {
- reporter.setStatus(getCountString());
- }
-
- /**
- * Return true if dst should be replaced by src and the update flag is set.
- * Right now, this merely checks that the src and dst len are not equal.
- * This should be improved on once modification times, CRCs, etc. can
- * be meaningful in this context.
- * @throws IOException
- */
- private boolean needsUpdate(FileStatus srcstatus,
- FileSystem dstfs, Path dstpath) throws IOException {
- return update && !sameFile(srcstatus.getPath().getFileSystem(job),
- srcstatus, dstfs, dstpath, skipCRCCheck);
- }
-
- private FSDataOutputStream create(Path f, Reporter reporter,
- FileStatus srcstat) throws IOException {
- if (destFileSys.exists(f)) {
- destFileSys.delete(f, false);
- }
- if (!preserve_status) {
- return destFileSys.create(f, true, sizeBuf, reporter);
- }
-
- FsPermission permission = preseved.contains(FileAttribute.PERMISSION)?
- srcstat.getPermission(): null;
- short replication = preseved.contains(FileAttribute.REPLICATION)?
- srcstat.getReplication(): destFileSys.getDefaultReplication(f);
- long blockSize = preseved.contains(FileAttribute.BLOCK_SIZE)?
- srcstat.getBlockSize(): destFileSys.getDefaultBlockSize(f);
- return destFileSys.create(f, permission, true, sizeBuf, replication,
- blockSize, reporter);
- }
-
- /**
- * Validates copy by checking the sizes of files first and then
- * checksums, if the filesystems support checksums.
- * @param srcstat src path and metadata
- * @param absdst dst path
- * @return true if src & destination files are same
- */
- private boolean validateCopy(FileStatus srcstat, Path absdst)
- throws IOException {
- if (destFileSys.exists(absdst)) {
- if (sameFile(srcstat.getPath().getFileSystem(job), srcstat,
- destFileSys, absdst, skipCRCCheck)) {
- return true;
- }
- }
- return false;
- }
-
- /**
- * Increment number of files copied and bytes copied and then report status
- */
- void updateCopyStatus(FileStatus srcstat, Reporter reporter) {
- copycount++;
- reporter.incrCounter(Counter.BYTESCOPIED, srcstat.getLen());
- reporter.incrCounter(Counter.COPY, 1);
- updateStatus(reporter);
- }
-
- /**
- * Skip copying this file if already exists at the destination.
- * Updates counters and copy status if skipping this file.
- * @return true if copy of this file can be skipped
- */
- private boolean skipCopyFile(FileStatus srcstat, Path absdst,
- OutputCollector<WritableComparable<?>, Text> outc,
- Reporter reporter) throws IOException {
- if (destFileSys.exists(absdst) && !overwrite
- && !needsUpdate(srcstat, destFileSys, absdst)) {
- outc.collect(null, new Text("SKIP: " + srcstat.getPath()));
- ++skipcount;
- reporter.incrCounter(Counter.SKIP, 1);
- updateStatus(reporter);
- return true;
- }
- return false;
- }
-
- /**
- * Copies single file to the path specified by tmpfile.
- * @param srcstat src path and metadata
- * @param tmpfile temporary file to which copy is to be done
- * @param absdst actual destination path to which copy is to be done
- * @param reporter
- * @return Number of bytes copied
- */
- private long doCopyFile(FileStatus srcstat, Path tmpfile, Path absdst,
- Reporter reporter) throws IOException {
- long bytesCopied = 0L;
- Path srcPath = srcstat.getPath();
- // open src file
- try (FSDataInputStream in = srcPath.getFileSystem(job).open(srcPath)) {
- reporter.incrCounter(Counter.BYTESEXPECTED, srcstat.getLen());
- // open tmp file
- try (FSDataOutputStream out = create(tmpfile, reporter, srcstat)) {
- LOG.info("Copying file " + srcPath + " of size " +
- srcstat.getLen() + " bytes...");
-
- // copy file
- for(int bytesRead; (bytesRead = in.read(buffer)) >= 0; ) {
- out.write(buffer, 0, bytesRead);
- bytesCopied += bytesRead;
- reporter.setStatus(
- String.format("%.2f ", bytesCopied*100.0/srcstat.getLen())
- + absdst + " [ " +
- TraditionalBinaryPrefix.long2String(bytesCopied, "", 1) + " / "
- + TraditionalBinaryPrefix.long2String(srcstat.getLen(), "", 1)
- + " ]");
- }
- }
- }
- return bytesCopied;
- }
-
- /**
- * Copy a file to a destination.
- * @param srcstat src path and metadata
- * @param relativedst relative dst path
- * @param outc Log of skipped files
- * @param reporter
- * @throws IOException if copy fails(even if the validation of copy fails)
- */
- private void copy(FileStatus srcstat, Path relativedst,
- OutputCollector<WritableComparable<?>, Text> outc, Reporter reporter)
- throws IOException {
- Path absdst = new Path(destPath, relativedst);
- int totfiles = job.getInt(SRC_COUNT_LABEL, -1);
- assert totfiles >= 0 : "Invalid file count " + totfiles;
-
- if (totfiles == 1) {
- // Copying a single file; use dst path provided by user as
- // destination file rather than destination directory
- Path dstparent = absdst.getParent();
- if (!(destFileSys.exists(dstparent) &&
- destFileSys.getFileStatus(dstparent).isDirectory())) {
- absdst = dstparent;
- }
- }
-
- // if a directory, ensure created even if empty
- if (srcstat.isDirectory()) {
- if (destFileSys.exists(absdst)) {
- if (destFileSys.getFileStatus(absdst).isFile()) {
- throw new IOException("Failed to mkdirs: " + absdst+" is a file.");
- }
- }
- else if (!destFileSys.mkdirs(absdst)) {
- throw new IOException("Failed to mkdirs " + absdst);
- }
- // TODO: when modification times can be set, directories should be
- // emitted to reducers so they might be preserved. Also, mkdirs does
- // not currently return an error when the directory already exists;
- // if this changes, all directory work might as well be done in reduce
- return;
- }
-
- // Can we skip copying this file ?
- if (skipCopyFile(srcstat, absdst, outc, reporter)) {
- return;
- }
-
- Path tmpfile = new Path(job.get(TMP_DIR_LABEL), relativedst);
- // do the actual copy to tmpfile
- long bytesCopied = doCopyFile(srcstat, tmpfile, absdst, reporter);
-
- if (bytesCopied != srcstat.getLen()) {
- throw new IOException("File size not matched: copied "
- + bytesString(bytesCopied) + " to tmpfile (=" + tmpfile
- + ") but expected " + bytesString(srcstat.getLen())
- + " from " + srcstat.getPath());
- }
- else {
- if (destFileSys.exists(absdst) &&
- destFileSys.getFileStatus(absdst).isDirectory()) {
- throw new IOException(absdst + " is a directory");
- }
- if (!destFileSys.mkdirs(absdst.getParent())) {
- throw new IOException("Failed to create parent dir: " + absdst.getParent());
- }
- rename(tmpfile, absdst);
-
- if (!validateCopy(srcstat, absdst)) {
- destFileSys.delete(absdst, false);
- throw new IOException("Validation of copy of file "
- + srcstat.getPath() + " failed.");
- }
- updateDestStatus(srcstat, destFileSys.getFileStatus(absdst));
- }
-
- // report at least once for each file
- updateCopyStatus(srcstat, reporter);
- }
-
- /** rename tmp to dst, delete dst if already exists */
- private void rename(Path tmp, Path dst) throws IOException {
- try {
- if (destFileSys.exists(dst)) {
- destFileSys.delete(dst, true);
- }
- if (!destFileSys.rename(tmp, dst)) {
- throw new IOException();
- }
- }
- catch(IOException cause) {
- throw (IOException)new IOException("Fail to rename tmp file (=" + tmp
- + ") to destination file (=" + dst + ")").initCause(cause);
- }
- }
-
- private void updateDestStatus(FileStatus src, FileStatus dst
- ) throws IOException {
- if (preserve_status) {
- DistCpV1.updateDestStatus(src, dst, preseved, destFileSys);
- }
- }
-
- static String bytesString(long b) {
- return b + " bytes (" +
- TraditionalBinaryPrefix.long2String(b, "", 1) + ")";
- }
-
- /**
- * Copies a file and validates the copy by checking the checksums.
- * If validation fails, retries (max number of tries is distcp.file.retries)
- * to copy the file.
- */
- void copyWithRetries(FileStatus srcstat, Path relativedst,
- OutputCollector<WritableComparable<?>, Text> out,
- Reporter reporter) throws IOException {
-
- // max tries to copy when validation of copy fails
- final int maxRetries = job.getInt(FILE_RETRIES_LABEL, DEFAULT_FILE_RETRIES);
- // save update flag for later copies within the same map task
- final boolean saveUpdate = update;
-
- int retryCnt = 1;
- for (; retryCnt <= maxRetries; retryCnt++) {
- try {
- //copy the file and validate copy
- copy(srcstat, relativedst, out, reporter);
- break;// copy successful
- } catch (IOException e) {
- LOG.warn("Copy of " + srcstat.getPath() + " failed.", e);
- if (retryCnt < maxRetries) {// copy failed and need to retry
- LOG.info("Retrying copy of file " + srcstat.getPath());
- update = true; // set update flag for retries
- }
- else {// no more retries... Give up
- update = saveUpdate;
- throw new IOException("Copy of file failed even with " + retryCnt
- + " tries.", e);
- }
- }
- }
- }
-
- /** Mapper configuration.
- * Extracts source and destination file system, as well as
- * top-level paths on source and destination directories.
- * Gets the named file systems, to be used later in map.
- */
- public void configure(JobConf job)
- {
- destPath = new Path(job.get(DST_DIR_LABEL, "/"));
- try {
- destFileSys = destPath.getFileSystem(job);
- } catch (IOException ex) {
- throw new RuntimeException("Unable to get the named file system.", ex);
- }
- sizeBuf = job.getInt("copy.buf.size", 128 * 1024);
- buffer = new byte[sizeBuf];
- ignoreReadFailures = job.getBoolean(Options.IGNORE_READ_FAILURES.propertyname, false);
- preserve_status = job.getBoolean(Options.PRESERVE_STATUS.propertyname, false);
- if (preserve_status) {
- preseved = FileAttribute.parse(job.get(PRESERVE_STATUS_LABEL));
- }
- update = job.getBoolean(Options.UPDATE.propertyname, false);
- overwrite = !update && job.getBoolean(Options.OVERWRITE.propertyname, false);
- skipCRCCheck = job.getBoolean(Options.SKIPCRC.propertyname, false);
- this.job = job;
- }
-
- /** Map method. Copies one file from source file system to destination.
- * @param key src len
- * @param value FilePair (FileStatus src, Path dst)
- * @param out Log of failed copies
- * @param reporter
- */
- public void map(LongWritable key,
- FilePair value,
- OutputCollector<WritableComparable<?>, Text> out,
- Reporter reporter) throws IOException {
- final FileStatus srcstat = value.input;
- final Path relativedst = new Path(value.output);
- try {
- copyWithRetries(srcstat, relativedst, out, reporter);
- } catch (IOException e) {
- ++failcount;
- reporter.incrCounter(Counter.FAIL, 1);
- updateStatus(reporter);
- final String sfailure = "FAIL " + relativedst + " : " +
- StringUtils.stringifyException(e);
- out.collect(null, new Text(sfailure));
- LOG.info(sfailure);
- if (e instanceof FileNotFoundException) {
- final String s = "Possible Cause for failure: Either the filesystem "
- + srcstat.getPath().getFileSystem(job)
- + " is not accessible or the file is deleted";
- LOG.error(s);
- out.collect(null, new Text(s));
- }
-
- try {
- for (int i = 0; i < 3; ++i) {
- try {
- final Path tmp = new Path(job.get(TMP_DIR_LABEL), relativedst);
- if (destFileSys.delete(tmp, true))
- break;
- } catch (Throwable ex) {
- // ignore, we are just cleaning up
- LOG.debug("Ignoring cleanup exception", ex);
- }
- // update status, so we don't get timed out
- updateStatus(reporter);
- Thread.sleep(3 * 1000);
- }
- } catch (InterruptedException inte) {
- throw (IOException)new IOException().initCause(inte);
- }
- } finally {
- updateStatus(reporter);
- }
- }
-
- public void close() throws IOException {
- if (0 == failcount || ignoreReadFailures) {
- return;
- }
- throw new IOException(getCountString());
- }
- }
-
- private static List<Path> fetchFileList(Configuration conf, Path srcList)
- throws IOException {
- List<Path> result = new ArrayList<Path>();
- FileSystem fs = srcList.getFileSystem(conf);
- try (BufferedReader input = new BufferedReader(new InputStreamReader(fs.open(srcList),
- Charset.forName("UTF-8")))) {
- String line = input.readLine();
- while (line != null) {
- result.add(new Path(line));
- line = input.readLine();
- }
- }
- return result;
- }
-
- @Deprecated
- public static void copy(Configuration conf, String srcPath,
- String destPath, Path logPath,
- boolean srcAsList, boolean ignoreReadFailures)
- throws IOException {
- final Path src = new Path(srcPath);
- List<Path> tmp = new ArrayList<Path>();
- if (srcAsList) {
- tmp.addAll(fetchFileList(conf, src));
- } else {
- tmp.add(src);
- }
- EnumSet<Options> flags = ignoreReadFailures
- ? EnumSet.of(Options.IGNORE_READ_FAILURES)
- : EnumSet.noneOf(Options.class);
-
- final Path dst = new Path(destPath);
- copy(conf, new Arguments(tmp, null, dst, logPath, flags, null,
- Long.MAX_VALUE, Long.MAX_VALUE, null, false));
- }
-
- /** Sanity check for srcPath */
- private static void checkSrcPath(JobConf jobConf, List<Path> srcPaths)
- throws IOException {
- List<IOException> rslt = new ArrayList<IOException>();
- List<Path> unglobbed = new LinkedList<Path>();
-
- Path[] ps = new Path[srcPaths.size()];
- ps = srcPaths.toArray(ps);
- TokenCache.obtainTokensForNamenodes(jobConf.getCredentials(), ps, jobConf);
-
-
- for (Path p : srcPaths) {
- FileSystem fs = p.getFileSystem(jobConf);
- FileStatus[] inputs = fs.globStatus(p);
-
- if(inputs != null && inputs.length > 0) {
- for (FileStatus onePath: inputs) {
- unglobbed.add(onePath.getPath());
- }
- } else {
- rslt.add(new IOException("Input source " + p + " does not exist."));
- }
- }
- if (!rslt.isEmpty()) {
- throw new InvalidInputException(rslt);
- }
- srcPaths.clear();
- srcPaths.addAll(unglobbed);
- }
-
- /**
- * Driver to copy srcPath to destPath depending on required protocol.
- * @param conf configuration
- * @param args arguments
- */
- static void copy(final Configuration conf, final Arguments args
- ) throws IOException {
- LOG.info("srcPaths=" + args.srcs);
- if (!args.dryrun || args.flags.contains(Options.UPDATE)) {
- LOG.info("destPath=" + args.dst);
- }
-
- JobConf job = createJobConf(conf);
-
- checkSrcPath(job, args.srcs);
- if (args.preservedAttributes != null) {
- job.set(PRESERVE_STATUS_LABEL, args.preservedAttributes);
- }
- if (args.mapredSslConf != null) {
- job.set("dfs.https.client.keystore.resource", args.mapredSslConf);
- }
-
- //Initialize the mapper
- try {
- if (setup(conf, job, args)) {
- JobClient.runJob(job);
- }
- if(!args.dryrun) {
- finalize(conf, job, args.dst, args.preservedAttributes);
- }
- } finally {
- if (!args.dryrun) {
- //delete tmp
- fullyDelete(job.get(TMP_DIR_LABEL), job);
- }
- //delete jobDirectory
- fullyDelete(job.get(JOB_DIR_LABEL), job);
- }
- }
-
- private static void updateDestStatus(FileStatus src, FileStatus dst,
- EnumSet<FileAttribute> preseved, FileSystem destFileSys
- ) throws IOException {
- String owner = null;
- String group = null;
- if (preseved.contains(FileAttribute.USER)
- && !src.getOwner().equals(dst.getOwner())) {
- owner = src.getOwner();
- }
- if (preseved.contains(FileAttribute.GROUP)
- && !src.getGroup().equals(dst.getGroup())) {
- group = src.getGroup();
- }
- if (owner != null || group != null) {
- destFileSys.setOwner(dst.getPath(), owner, group);
- }
- if (preseved.contains(FileAttribute.PERMISSION)
- && !src.getPermission().equals(dst.getPermission())) {
- destFileSys.setPermission(dst.getPath(), src.getPermission());
- }
- if (preseved.contains(FileAttribute.TIMES)) {
- destFileSys.setTimes(dst.getPath(), src.getModificationTime(), src.getAccessTime());
- }
- }
-
- static private void finalize(Configuration conf, JobConf jobconf,
- final Path destPath, String presevedAttributes) throws IOException {
- if (presevedAttributes == null) {
- return;
- }
- EnumSet<FileAttribute> preseved = FileAttribute.parse(presevedAttributes);
- if (!preseved.contains(FileAttribute.USER)
- && !preseved.contains(FileAttribute.GROUP)
- && !preseved.contains(FileAttribute.PERMISSION)) {
- return;
- }
-
- FileSystem dstfs = destPath.getFileSystem(conf);
- Path dstdirlist = new Path(jobconf.get(DST_DIR_LIST_LABEL));
- try (SequenceFile.Reader in =
- new SequenceFile.Reader(jobconf, Reader.file(dstdirlist))) {
- Text dsttext = new Text();
- FilePair pair = new FilePair();
- for(; in.next(dsttext, pair); ) {
- Path absdst = new Path(destPath, pair.output);
- updateDestStatus(pair.input, dstfs.getFileStatus(absdst),
- preseved, dstfs);
- }
- }
- }
-
- static class Arguments {
- final List<Path> srcs;
- final Path basedir;
- final Path dst;
- final Path log;
- final EnumSet<Options> flags;
- final String preservedAttributes;
- final long filelimit;
- final long sizelimit;
- final String mapredSslConf;
- final boolean dryrun;
-
- /**
- * Arguments for distcp
- * @param srcs List of source paths
- * @param basedir Base directory for copy
- * @param dst Destination path
- * @param log Log output directory
- * @param flags Command-line flags
- * @param preservedAttributes Preserved attributes
- * @param filelimit File limit
- * @param sizelimit Size limit
- * @param mapredSslConf ssl configuration
- * @param dryrun
- */
- Arguments(List<Path> srcs, Path basedir, Path dst, Path log,
- EnumSet<Options> flags, String preservedAttributes,
- long filelimit, long sizelimit, String mapredSslConf,
- boolean dryrun) {
- this.srcs = srcs;
- this.basedir = basedir;
- this.dst = dst;
- this.log = log;
- this.flags = flags;
- this.preservedAttributes = preservedAttributes;
- this.filelimit = filelimit;
- this.sizelimit = sizelimit;
- this.mapredSslConf = mapredSslConf;
- this.dryrun = dryrun;
-
- if (LOG.isTraceEnabled()) {
- LOG.trace("this = " + this);
- }
- }
-
- static Arguments valueOf(String[] args, Configuration conf
- ) throws IOException {
- List<Path> srcs = new ArrayList<Path>();
- Path dst = null;
- Path log = null;
- Path basedir = null;
- EnumSet<Options> flags = EnumSet.noneOf(Options.class);
- String presevedAttributes = null;
- String mapredSslConf = null;
- long filelimit = Long.MAX_VALUE;
- long sizelimit = Long.MAX_VALUE;
- boolean dryrun = false;
-
- for (int idx = 0; idx < args.length; idx++) {
- Options[] opt = Options.values();
- int i = 0;
- for(; i < opt.length && !args[idx].startsWith(opt[i].cmd); i++);
-
- if (i < opt.length) {
- flags.add(opt[i]);
- if (opt[i] == Options.PRESERVE_STATUS) {
- presevedAttributes = args[idx].substring(2);
- FileAttribute.parse(presevedAttributes); //validation
- }
- else if (opt[i] == Options.FILE_LIMIT) {
- filelimit = Options.FILE_LIMIT.parseLong(args, ++idx);
- }
- else if (opt[i] == Options.SIZE_LIMIT) {
- sizelimit = Options.SIZE_LIMIT.parseLong(args, ++idx);
- }
- } else if ("-f".equals(args[idx])) {
- if (++idx == args.length) {
- throw new IllegalArgumentException("urilist_uri not specified in -f");
- }
- srcs.addAll(fetchFileList(conf, new Path(args[idx])));
- } else if ("-log".equals(args[idx])) {
- if (++idx == args.length) {
- throw new IllegalArgumentException("logdir not specified in -log");
- }
- log = new Path(args[idx]);
- } else if ("-basedir".equals(args[idx])) {
- if (++idx == args.length) {
- throw new IllegalArgumentException("basedir not specified in -basedir");
- }
- basedir = new Path(args[idx]);
- } else if ("-mapredSslConf".equals(args[idx])) {
- if (++idx == args.length) {
- throw new IllegalArgumentException("ssl conf file not specified in -mapredSslConf");
- }
- mapredSslConf = args[idx];
- } else if ("-dryrun".equals(args[idx])) {
- dryrun = true;
- dst = new Path("/tmp/distcp_dummy_dest");//dummy destination
- } else if ("-m".equals(args[idx])) {
- if (++idx == args.length) {
- throw new IllegalArgumentException("num_maps not specified in -m");
- }
- try {
- conf.setInt(MAX_MAPS_LABEL, Integer.parseInt(args[idx]));
- } catch (NumberFormatException e) {
- throw new IllegalArgumentException("Invalid argument to -m: " +
- args[idx]);
- }
- } else if ('-' == args[idx].codePointAt(0)) {
- throw new IllegalArgumentException("Invalid switch " + args[idx]);
- } else if (idx == args.length -1 &&
- (!dryrun || flags.contains(Options.UPDATE))) {
- dst = new Path(args[idx]);
- } else {
- srcs.add(new Path(args[idx]));
- }
- }
- // mandatory command-line parameters
- if (srcs.isEmpty() || dst == null) {
- throw new IllegalArgumentException("Missing "
- + (dst == null ? "dst path" : "src"));
- }
- // incompatible command-line flags
- final boolean isOverwrite = flags.contains(Options.OVERWRITE);
- final boolean isUpdate = flags.contains(Options.UPDATE);
- final boolean isDelete = flags.contains(Options.DELETE);
- final boolean skipCRC = flags.contains(Options.SKIPCRC);
- if (isOverwrite && isUpdate) {
- throw new IllegalArgumentException("Conflicting overwrite policies");
- }
- if (!isUpdate && skipCRC) {
- throw new IllegalArgumentException(
- Options.SKIPCRC.cmd + " is relevant only with the " +
- Options.UPDATE.cmd + " option");
- }
- if (isDelete && !isOverwrite && !isUpdate) {
- throw new IllegalArgumentException(Options.DELETE.cmd
- + " must be specified with " + Options.OVERWRITE + " or "
- + Options.UPDATE + ".");
- }
- return new Arguments(srcs, basedir, dst, log, flags, presevedAttributes,
- filelimit, sizelimit, mapredSslConf, dryrun);
- }
-
- /** {@inheritDoc} */
- public String toString() {
- return getClass().getName() + "{"
- + "\n srcs = " + srcs
- + "\n dst = " + dst
- + "\n log = " + log
- + "\n flags = " + flags
- + "\n preservedAttributes = " + preservedAttributes
- + "\n filelimit = " + filelimit
- + "\n sizelimit = " + sizelimit
- + "\n mapredSslConf = " + mapredSslConf
- + "\n}";
- }
- }
-
- /**
- * This is the main driver for recursively copying directories
- * across file systems. It takes at least two cmdline parameters. A source
- * URL and a destination URL. It then essentially does an "ls -lR" on the
- * source URL, and writes the output in a round-robin manner to all the map
- * input files. The mapper actually copies the files allotted to it. The
- * reduce is empty.
- */
- public int run(String[] args) {
- try {
- copy(conf, Arguments.valueOf(args, conf));
- return 0;
- } catch (IllegalArgumentException e) {
- System.err.println(StringUtils.stringifyException(e) + "\n" + usage);
- ToolRunner.printGenericCommandUsage(System.err);
- return -1;
- } catch (DuplicationException e) {
- System.err.println(StringUtils.stringifyException(e));
- return DuplicationException.ERROR_CODE;
- } catch (RemoteException e) {
- final IOException unwrapped = e.unwrapRemoteException(
- FileNotFoundException.class,
- AccessControlException.class,
- QuotaExceededException.class);
- System.err.println(StringUtils.stringifyException(unwrapped));
- return -3;
- } catch (Exception e) {
- System.err.println("With failures, global counters are inaccurate; " +
- "consider running with -i");
- System.err.println("Copy failed: " + StringUtils.stringifyException(e));
- return -999;
- }
- }
-
- public static void main(String[] args) throws Exception {
- JobConf job = new JobConf(DistCpV1.class);
- DistCpV1 distcp = new DistCpV1(job);
- int res = ToolRunner.run(distcp, args);
- System.exit(res);
- }
-
- /**
- * Make a path relative with respect to a root path.
- * absPath is always assumed to descend from root.
- * Otherwise returned path is null.
- */
- static String makeRelative(Path root, Path absPath) {
- if (!absPath.isAbsolute()) {
- throw new IllegalArgumentException("!absPath.isAbsolute(), absPath="
- + absPath);
- }
- String p = absPath.toUri().getPath();
-
- StringTokenizer pathTokens = new StringTokenizer(p, "/");
- for(StringTokenizer rootTokens = new StringTokenizer(
- root.toUri().getPath(), "/"); rootTokens.hasMoreTokens(); ) {
- if (!rootTokens.nextToken().equals(pathTokens.nextToken())) {
- return null;
- }
- }
- StringBuilder sb = new StringBuilder();
- for(; pathTokens.hasMoreTokens(); ) {
- sb.append(pathTokens.nextToken());
- if (pathTokens.hasMoreTokens()) { sb.append(Path.SEPARATOR); }
- }
- return sb.length() == 0? ".": sb.toString();
- }
-
- /**
- * Calculate how many maps to run.
- * Number of maps is bounded by a minimum of the cumulative size of the
- * copy / (distcp.bytes.per.map, default BYTES_PER_MAP or -m on the
- * command line) and at most (distcp.max.map.tasks, default
- * MAX_MAPS_PER_NODE * nodes in the cluster).
- * @param totalBytes Count of total bytes for job
- * @param job The job to configure
- * @return Count of maps to run.
- */
- private static int setMapCount(long totalBytes, JobConf job)
- throws IOException {
- int numMaps =
- (int)(totalBytes / job.getLong(BYTES_PER_MAP_LABEL, BYTES_PER_MAP));
- numMaps = Math.min(numMaps,
- job.getInt(MAX_MAPS_LABEL, MAX_MAPS_PER_NODE *
- new JobClient(job).getClusterStatus().getTaskTrackers()));
- numMaps = Math.max(numMaps, 1);
- job.setNumMapTasks(numMaps);
- return numMaps;
- }
-
- /** Fully delete dir */
- static void fullyDelete(String dir, Configuration conf) throws IOException {
- if (dir != null) {
- Path tmp = new Path(dir);
- boolean success = tmp.getFileSystem(conf).delete(tmp, true);
- if (!success) {
- LOG.warn("Could not fully delete " + tmp);
- }
- }
- }
-
- //Job configuration
- private static JobConf createJobConf(Configuration conf) {
- JobConf jobconf = new JobConf(conf, DistCpV1.class);
- jobconf.setJobName(conf.get("mapred.job.name", NAME));
-
- // turn off speculative execution, because DFS doesn't handle
- // multiple writers to the same file.
- jobconf.setMapSpeculativeExecution(false);
-
- jobconf.setInputFormat(CopyInputFormat.class);
- jobconf.setOutputKeyClass(Text.class);
- jobconf.setOutputValueClass(Text.class);
-
- jobconf.setMapperClass(CopyFilesMapper.class);
- jobconf.setNumReduceTasks(0);
- return jobconf;
- }
-
- private static final Random RANDOM = new Random();
- public static String getRandomId() {
- return Integer.toString(RANDOM.nextInt(Integer.MAX_VALUE), 36);
- }
-
- /**
- * Increase the replication factor of _distcp_src_files to
- * sqrt(min(maxMapsOnCluster, numMaps)). This is to reduce the chance of
- * failing of distcp because of "not having a replication of _distcp_src_files
- * available for reading for some maps".
- */
- private static void setReplication(Configuration conf, JobConf jobConf,
- Path srcfilelist, int numMaps) throws IOException {
- int numMaxMaps = new JobClient(jobConf).getClusterStatus().getMaxMapTasks();
- short replication = (short) Math.ceil(
- Math.sqrt(Math.min(numMaxMaps, numMaps)));
- FileSystem fs = srcfilelist.getFileSystem(conf);
- FileStatus srcStatus = fs.getFileStatus(srcfilelist);
-
- if (srcStatus.getReplication() < replication) {
- if (!fs.setReplication(srcfilelist, replication)) {
- throw new IOException("Unable to increase the replication of file " +
- srcfilelist);
- }
- }
- }
-
- /**
- * Does the dir already exist at destination ?
- * @return true if the dir already exists at destination
- */
- private static boolean dirExists(Configuration conf, Path dst)
- throws IOException {
- FileSystem destFileSys = dst.getFileSystem(conf);
- FileStatus status = null;
- try {
- status = destFileSys.getFileStatus(dst);
- }catch (FileNotFoundException e) {
- return false;
- }
- if (status.isFile()) {
- throw new FileAlreadyExistsException("Not a dir: " + dst+" is a file.");
- }
- return true;
- }
-
- /**
- * Initialize DFSCopyFileMapper specific job-configuration.
- * @param conf : The dfs/mapred configuration.
- * @param jobConf : The handle to the jobConf object to be initialized.
- * @param args Arguments
- * @return true if it is necessary to launch a job.
- */
- static boolean setup(Configuration conf, JobConf jobConf,
- final Arguments args)
- throws IOException {
- jobConf.set(DST_DIR_LABEL, args.dst.toUri().toString());
-
- //set boolean values
- final boolean update = args.flags.contains(Options.UPDATE);
- final boolean skipCRCCheck = args.flags.contains(Options.SKIPCRC);
- final boolean overwrite = !update && args.flags.contains(Options.OVERWRITE)
- && !args.dryrun;
- jobConf.setBoolean(Options.UPDATE.propertyname, update);
- jobConf.setBoolean(Options.SKIPCRC.propertyname, skipCRCCheck);
- jobConf.setBoolean(Options.OVERWRITE.propertyname, overwrite);
- jobConf.setBoolean(Options.IGNORE_READ_FAILURES.propertyname,
- args.flags.contains(Options.IGNORE_READ_FAILURES));
- jobConf.setBoolean(Options.PRESERVE_STATUS.propertyname,
- args.flags.contains(Options.PRESERVE_STATUS));
-
- final String randomId = getRandomId();
- JobClient jClient = new JobClient(jobConf);
- Path stagingArea;
- try {
- stagingArea =
- JobSubmissionFiles.getStagingDir(jClient.getClusterHandle(), conf);
- } catch (InterruptedException ie) {
- throw new IOException(ie);
- }
-
- Path jobDirectory = new Path(stagingArea + NAME + "_" + randomId);
- FsPermission mapredSysPerms =
- new FsPermission(JobSubmissionFiles.JOB_DIR_PERMISSION);
- FileSystem.mkdirs(jClient.getFs(), jobDirectory, mapredSysPerms);
- jobConf.set(JOB_DIR_LABEL, jobDirectory.toString());
-
- long maxBytesPerMap = conf.getLong(BYTES_PER_MAP_LABEL, BYTES_PER_MAP);
-
- FileSystem dstfs = args.dst.getFileSystem(conf);
-
- // get tokens for all the required FileSystems..
- TokenCache.obtainTokensForNamenodes(jobConf.getCredentials(),
- new Path[] {args.dst}, conf);
-
-
- boolean dstExists = dstfs.exists(args.dst);
- boolean dstIsDir = false;
- if (dstExists) {
- dstIsDir = dstfs.getFileStatus(args.dst).isDirectory();
- }
-
- // default logPath
- Path logPath = args.log;
- if (logPath == null) {
- String filename = "_distcp_logs_" + randomId;
- if (!dstExists || !dstIsDir) {
- Path parent = args.dst.getParent();
- if (null == parent) {
- // If dst is '/' on S3, it might not exist yet, but dst.getParent()
- // will return null. In this case, use '/' as its own parent to prevent
- // NPE errors below.
- parent = args.dst;
- }
- if (!dstfs.exists(parent)) {
- dstfs.mkdirs(parent);
- }
- logPath = new Path(parent, filename);
- } else {
- logPath = new Path(args.dst, filename);
- }
- }
- FileOutputFormat.setOutputPath(jobConf, logPath);
-
- // create src list, dst list
- FileSystem jobfs = jobDirectory.getFileSystem(jobConf);
-
- Path srcfilelist = new Path(jobDirectory, "_distcp_src_files");
- Path dstfilelist = new Path(jobDirectory, "_distcp_dst_files");
- Path dstdirlist = new Path(jobDirectory, "_distcp_dst_dirs");
- jobConf.set(SRC_LIST_LABEL, srcfilelist.toString());
- jobConf.set(DST_DIR_LIST_LABEL, dstdirlist.toString());
- int srcCount = 0, cnsyncf = 0, dirsyn = 0;
- long fileCount = 0L, dirCount = 0L, byteCount = 0L, cbsyncs = 0L,
- skipFileCount = 0L, skipByteCount = 0L;
- try (
- SequenceFile.Writer src_writer = SequenceFile.createWriter(jobConf,
- Writer.file(srcfilelist), Writer.keyClass(LongWritable.class),
- Writer.valueClass(FilePair.class), Writer.compression(
- SequenceFile.CompressionType.NONE));
- SequenceFile.Writer dst_writer = SequenceFile.createWriter(jobConf,
- Writer.file(dstfilelist), Writer.keyClass(Text.class),
- Writer.valueClass(Text.class), Writer.compression(
- SequenceFile.CompressionType.NONE));
- SequenceFile.Writer dir_writer = SequenceFile.createWriter(jobConf,
- Writer.file(dstdirlist), Writer.keyClass(Text.class),
- Writer.valueClass(FilePair.class), Writer.compression(
- SequenceFile.CompressionType.NONE));
- ) {
- // handle the case where the destination directory doesn't exist
- // and we've only a single src directory OR we're updating/overwriting
- // the contents of the destination directory.
- final boolean special =
- (args.srcs.size() == 1 && !dstExists) || update || overwrite;
-
- Path basedir = null;
- HashSet<Path> parentDirsToCopy = new HashSet<Path>();
- if (args.basedir != null) {
- FileSystem basefs = args.basedir.getFileSystem(conf);
- basedir = args.basedir.makeQualified(
- basefs.getUri(), basefs.getWorkingDirectory());
- if (!basefs.isDirectory(basedir)) {
- throw new IOException("Basedir " + basedir + " is not a directory.");
- }
- }
-
- for(Iterator<Path> srcItr = args.srcs.iterator(); srcItr.hasNext(); ) {
- final Path src = srcItr.next();
- FileSystem srcfs = src.getFileSystem(conf);
- FileStatus srcfilestat = srcfs.getFileStatus(src);
- Path root = special && srcfilestat.isDirectory()? src: src.getParent();
- if (dstExists && !dstIsDir &&
- (args.srcs.size() > 1 || srcfilestat.isDirectory())) {
- // destination should not be a file
- throw new IOException("Destination " + args.dst + " should be a dir" +
- " if multiple source paths are there OR if" +
- " the source path is a dir");
- }
-
- if (basedir != null) {
- root = basedir;
- Path parent = src.getParent().makeQualified(
- srcfs.getUri(), srcfs.getWorkingDirectory());
- while (parent != null && !parent.equals(basedir)) {
- if (!parentDirsToCopy.contains(parent)){
- parentDirsToCopy.add(parent);
- String dst = makeRelative(root, parent);
- FileStatus pst = srcfs.getFileStatus(parent);
- src_writer.append(new LongWritable(0), new FilePair(pst, dst));
- dst_writer.append(new Text(dst), new Text(parent.toString()));
- dir_writer.append(new Text(dst), new FilePair(pst, dst));
- if (++dirsyn > SYNC_FILE_MAX) {
- dirsyn = 0;
- dir_writer.sync();
- }
- }
- parent = parent.getParent();
- }
-
- if (parent == null) {
- throw new IOException("Basedir " + basedir +
- " is not a prefix of source path " + src);
- }
- }
-
- if (srcfilestat.isDirectory()) {
- ++srcCount;
- final String dst = makeRelative(root,src);
- if (!update || !dirExists(conf, new Path(args.dst, dst))) {
- ++dirCount;
- src_writer.append(new LongWritable(0),
- new FilePair(srcfilestat, dst));
- }
- dst_writer.append(new Text(dst), new Text(src.toString()));
- }
-
- Stack<FileStatus> pathstack = new Stack<FileStatus>();
- for(pathstack.push(srcfilestat); !pathstack.empty(); ) {
- FileStatus cur = pathstack.pop();
- FileStatus[] children = srcfs.listStatus(cur.getPath());
- for(int i = 0; i < children.length; i++) {
- boolean skipPath = false;
- final FileStatus child = children[i];
- final String dst = makeRelative(root, child.getPath());
- ++srcCount;
-
- if (child.isDirectory()) {
- pathstack.push(child);
- if (!update || !dirExists(conf, new Path(args.dst, dst))) {
- ++dirCount;
- }
- else {
- skipPath = true; // skip creating dir at destination
- }
- }
- else {
- Path destPath = new Path(args.dst, dst);
- if (cur.isFile() && (args.srcs.size() == 1)) {
- // Copying a single file; use dst path provided by user as
- // destination file rather than destination directory
- Path dstparent = destPath.getParent();
- FileSystem destFileSys = destPath.getFileSystem(jobConf);
- if (!(destFileSys.exists(dstparent) &&
- destFileSys.getFileStatus(dstparent).isDirectory())) {
- destPath = dstparent;
- }
- }
- //skip path if the src and the dst files are the same.
- skipPath = update &&
- sameFile(srcfs, child, dstfs, destPath, skipCRCCheck);
- //skip path if it exceed file limit or size limit
- skipPath |= fileCount == args.filelimit
- || byteCount + child.getLen() > args.sizelimit;
-
- if (!skipPath) {
- ++fileCount;
- byteCount += child.getLen();
-
- if (LOG.isTraceEnabled()) {
- LOG.trace("adding file " + child.getPath());
- }
-
- ++cnsyncf;
- cbsyncs += child.getLen();
- if (cnsyncf > SYNC_FILE_MAX || cbsyncs > maxBytesPerMap) {
- src_writer.sync();
- dst_writer.sync();
- cnsyncf = 0;
- cbsyncs = 0L;
- }
- }
- else {
- ++skipFileCount;
- skipByteCount += child.getLen();
- if (LOG.isTraceEnabled()) {
- LOG.trace("skipping file " + child.getPath());
- }
- }
- }
-
- if (!skipPath) {
- src_writer.append(new LongWritable(child.isDirectory()? 0: child.getLen()),
- new FilePair(child, dst));
- }
-
- dst_writer.append(new Text(dst),
- new Text(child.getPath().toString()));
- }
-
- if (cur.isDirectory()) {
- String dst = makeRelative(root, cur.getPath());
- dir_writer.append(new Text(dst), new FilePair(cur, dst));
- if (++dirsyn > SYNC_FILE_MAX) {
- dirsyn = 0;
- dir_writer.sync();
- }
- }
- }
- }
- }
- LOG.info("sourcePathsCount(files+directories)=" + srcCount);
- LOG.info("filesToCopyCount=" + fileCount);
- LOG.info("bytesToCopyCount=" +
- TraditionalBinaryPrefix.long2String(byteCount, "", 1));
- if (update) {
- LOG.info("filesToSkipCopyCount=" + skipFileCount);
- LOG.info("bytesToSkipCopyCount=" +
- TraditionalBinaryPrefix.long2String(skipByteCount, "", 1));
- }
- if (args.dryrun) {
- return false;
- }
- int mapCount = setMapCount(byteCount, jobConf);
- // Increase the replication of _distcp_src_files, if needed
- setReplication(conf, jobConf, srcfilelist, mapCount);
-
- FileStatus dststatus = null;
- try {
- dststatus = dstfs.getFileStatus(args.dst);
- } catch(FileNotFoundException fnfe) {
- LOG.info(args.dst + " does not exist.");
- }
-
- // create dest path dir if copying > 1 file
- if (dststatus == null) {
- if (srcCount > 1 && !dstfs.mkdirs(args.dst)) {
- throw new IOException("Failed to create" + args.dst);
- }
- }
-
- final Path sorted = new Path(jobDirectory, "_distcp_sorted");
- checkDuplication(jobfs, dstfilelist, sorted, conf);
-
- if (dststatus != null && args.flags.contains(Options.DELETE)) {
- long deletedPathsCount = deleteNonexisting(dstfs, dststatus, sorted,
- jobfs, jobDirectory, jobConf, conf);
- LOG.info("deletedPathsFromDestCount(files+directories)=" +
- deletedPathsCount);
- }
-
- Path tmpDir = new Path(
- (dstExists && !dstIsDir) || (!dstExists && srcCount == 1)?
- args.dst.getParent(): args.dst, "_distcp_tmp_" + randomId);
- jobConf.set(TMP_DIR_LABEL, tmpDir.toUri().toString());
-
- // Explicitly create the tmpDir to ensure that it can be cleaned
- // up by fullyDelete() later.
- tmpDir.getFileSystem(conf).mkdirs(tmpDir);
-
- LOG.info("sourcePathsCount=" + srcCount);
- LOG.info("filesToCopyCount=" + fileCount);
- LOG.info("bytesToCopyCount=" +
- TraditionalBinaryPrefix.long2String(byteCount, "", 1));
- jobConf.setInt(SRC_COUNT_LABEL, srcCount);
- jobConf.setLong(TOTAL_SIZE_LABEL, byteCount);
-
- return (fileCount + dirCount) > 0;
- }
-
- /**
- * Check whether the contents of src and dst are the same.
- *
- * Return false if dstpath does not exist
- *
- * If the files have different sizes, return false.
- *
- * If the files have the same sizes, the file checksums will be compared.
- *
- * When file checksum is not supported in any of file systems,
- * two files are considered as the same if they have the same size.
- */
- static private boolean sameFile(FileSystem srcfs, FileStatus srcstatus,
- FileSystem dstfs, Path dstpath, boolean skipCRCCheck) throws IOException {
- FileStatus dststatus;
- try {
- dststatus = dstfs.getFileStatus(dstpath);
- } catch(FileNotFoundException fnfe) {
- return false;
- }
-
- //same length?
- if (srcstatus.getLen() != dststatus.getLen()) {
- return false;
- }
-
- if (skipCRCCheck) {
- LOG.debug("Skipping the CRC check");
- return true;
- }
-
- //get src checksum
- final FileChecksum srccs;
- try {
- srccs = srcfs.getFileChecksum(srcstatus.getPath());
- } catch(FileNotFoundException fnfe) {
- /*
- * Two possible cases:
- * (1) src existed once but was deleted between the time period that
- * srcstatus was obtained and the try block above.
- * (2) srcfs does not support file checksum and (incorrectly) throws
- * FNFE, e.g. some previous versions of HftpFileSystem.
- * For case (1), it is okay to return true since src was already deleted.
- * For case (2), true should be returned.
- */
- return true;
- }
-
- //compare checksums
- try {
- final FileChecksum dstcs = dstfs.getFileChecksum(dststatus.getPath());
- //return true if checksum is not supported
- //(i.e. some of the checksums is null)
- return srccs == null || dstcs == null || srccs.equals(dstcs);
- } catch(FileNotFoundException fnfe) {
- return false;
- }
- }
-
- /**
- * Delete the dst files/dirs which do not exist in src
- *
- * @return total count of files and directories deleted from destination
- * @throws IOException
- */
- static private long deleteNonexisting(
- FileSystem dstfs, FileStatus dstroot, Path dstsorted,
- FileSystem jobfs, Path jobdir, JobConf jobconf, Configuration conf
- ) throws IOException {
- if (dstroot.isFile()) {
- throw new IOException("dst must be a directory when option "
- + Options.DELETE.cmd + " is set, but dst (= " + dstroot.getPath()
- + ") is not a directory.");
- }
-
- //write dst lsr results
- final Path dstlsr = new Path(jobdir, "_distcp_dst_lsr");
- try (final SequenceFile.Writer writer = SequenceFile.createWriter(jobconf,
- Writer.file(dstlsr), Writer.keyClass(Text.class),
- Writer.valueClass(NullWritable.class), Writer.compression(
- SequenceFile.CompressionType.NONE))) {
- //do lsr to get all file statuses in dstroot
- final Stack<FileStatus> lsrstack = new Stack<FileStatus>();
- for(lsrstack.push(dstroot); !lsrstack.isEmpty(); ) {
- final FileStatus status = lsrstack.pop();
- if (status.isDirectory()) {
- for(FileStatus child : dstfs.listStatus(status.getPath())) {
- String relative = makeRelative(dstroot.getPath(), child.getPath());
- writer.append(new Text(relative), NullWritable.get());
- lsrstack.push(child);
- }
- }
- }
- }
-
- //sort lsr results
- final Path sortedlsr = new Path(jobdir, "_distcp_dst_lsr_sorted");
- SequenceFile.Sorter sorter = new SequenceFile.Sorter(jobfs,
- new Text.Comparator(), Text.class, NullWritable.class, jobconf);
- sorter.sort(dstlsr, sortedlsr);
-
- //compare lsr list and dst list
- long deletedPathsCount = 0;
- try (SequenceFile.Reader lsrin =
- new SequenceFile.Reader(jobconf, Reader.file(sortedlsr));
- SequenceFile.Reader dstin =
- new SequenceFile.Reader(jobconf, Reader.file(dstsorted))) {
- //compare sorted lsr list and sorted dst list
- final Text lsrpath = new Text();
- final Text dstpath = new Text();
- final Text dstfrom = new Text();
- final Trash trash = new Trash(dstfs, conf);
- Path lastpath = null;
-
- boolean hasnext = dstin.next(dstpath, dstfrom);
- while (lsrin.next(lsrpath, NullWritable.get())) {
- int dst_cmp_lsr = dstpath.compareTo(lsrpath);
- while (hasnext && dst_cmp_lsr < 0) {
- hasnext = dstin.next(dstpath, dstfrom);
- dst_cmp_lsr = dstpath.compareTo(lsrpath);
- }
-
- if (dst_cmp_lsr == 0) {
- //lsrpath exists in dst, skip it
- hasnext = dstin.next(dstpath, dstfrom);
- } else {
- //lsrpath does not exist, delete it
- final Path rmpath = new Path(dstroot.getPath(), lsrpath.toString());
- ++deletedPathsCount;
- if ((lastpath == null || !isAncestorPath(lastpath, rmpath))) {
- if (!(trash.moveToTrash(rmpath) || dstfs.delete(rmpath, true))) {
- throw new IOException("Failed to delete " + rmpath);
- }
- lastpath = rmpath;
- }
- }
- }
- }
- return deletedPathsCount;
- }
-
- //is x an ancestor path of y?
- static private boolean isAncestorPath(Path xp, Path yp) {
- final String x = xp.toString();
- final String y = yp.toString();
- if (!y.startsWith(x)) {
- return false;
- }
- final int len = x.length();
- return y.length() == len || y.charAt(len) == Path.SEPARATOR_CHAR;
- }
-
- /** Check whether the file list have duplication. */
- static private void checkDuplication(FileSystem fs, Path file, Path sorted,
- Configuration conf) throws IOException {
- SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs,
- new Text.Comparator(), Text.class, Text.class, conf);
- sorter.sort(file, sorted);
- try (SequenceFile.Reader in =
- new SequenceFile.Reader(conf, Reader.file(sorted))) {
- Text prevdst = null, curdst = new Text();
- Text prevsrc = null, cursrc = new Text();
- for(; in.next(curdst, cursrc); ) {
- if (prevdst != null && curdst.equals(prevdst)) {
- throw new DuplicationException(
- "Invalid input, there are duplicated files in the sources: "
- + prevsrc + ", " + cursrc);
- }
- prevdst = curdst;
- curdst = new Text();
- prevsrc = cursrc;
- cursrc = new Text();
- }
- }
- }
-
- /** An exception class for duplicated source files. */
- public static class DuplicationException extends IOException {
- private static final long serialVersionUID = 1L;
- /** Error code for this exception */
- public static final int ERROR_CODE = -2;
- DuplicationException(String message) {super(message);}
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/850cc678/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java b/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java
deleted file mode 100644
index 05e6e24..0000000
--- a/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java
+++ /dev/null
@@ -1,329 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.tools;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.Random;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configuration.DeprecationDelta;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.io.WritableComparator;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MapReduceBase;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.TextInputFormat;
-import org.apache.hadoop.mapred.TextOutputFormat;
-import org.apache.hadoop.mapred.lib.LongSumReducer;
-import org.apache.hadoop.mapreduce.lib.map.RegexMapper;
-
-/**
- * Logalyzer: A utility tool for archiving and analyzing hadoop logs.
- * <p>
- * This tool supports archiving and anaylzing (sort/grep) of log-files.
- * It takes as input
- * a) Input uri which will serve uris of the logs to be archived.
- * b) Output directory (not mandatory).
- * b) Directory on dfs to archive the logs.
- * c) The sort/grep patterns for analyzing the files and separator for boundaries.
- * Usage:
- * Logalyzer -archive -archiveDir <directory to archive logs> -analysis
- * <directory> -logs <log-list uri> -grep <pattern> -sort
- * <col1, col2> -separator <separator>
- * <p>
- */
-@Deprecated
-public class Logalyzer {
- // Constants
- private static Configuration fsConfig = new Configuration();
- public static final String SORT_COLUMNS =
- "logalizer.logcomparator.sort.columns";
- public static final String COLUMN_SEPARATOR =
- "logalizer.logcomparator.column.separator";
-
- static {
- Configuration.addDeprecations(new DeprecationDelta[] {
- new DeprecationDelta("mapred.reducer.sort", SORT_COLUMNS),
- new DeprecationDelta("mapred.reducer.separator", COLUMN_SEPARATOR)
- });
- }
-
- /** A {@link Mapper} that extracts text matching a regular expression. */
- public static class LogRegexMapper<K extends WritableComparable>
- extends MapReduceBase
- implements Mapper<K, Text, Text, LongWritable> {
-
- private Pattern pattern;
-
- public void configure(JobConf job) {
- pattern = Pattern.compile(job.get(RegexMapper.PATTERN));
- }
-
- public void map(K key, Text value,
- OutputCollector<Text, LongWritable> output,
- Reporter reporter)
- throws IOException {
- String text = value.toString();
- Matcher matcher = pattern.matcher(text);
- while (matcher.find()) {
- output.collect(value, new LongWritable(1));
- }
- }
-
- }
-
- /** A WritableComparator optimized for UTF8 keys of the logs. */
- public static class LogComparator extends Text.Comparator implements Configurable {
-
- private static Log LOG = LogFactory.getLog(Logalyzer.class);
- private JobConf conf = null;
- private String[] sortSpec = null;
- private String columnSeparator = null;
-
- public void setConf(Configuration conf) {
- if (conf instanceof JobConf) {
- this.conf = (JobConf) conf;
- } else {
- this.conf = new JobConf(conf);
- }
-
- //Initialize the specification for *comparision*
- String sortColumns = this.conf.get(SORT_COLUMNS, null);
- if (sortColumns != null) {
- sortSpec = sortColumns.split(",");
- }
-
- //Column-separator
- columnSeparator = this.conf.get(COLUMN_SEPARATOR, "");
- }
-
- public Configuration getConf() {
- return conf;
- }
-
- public int compare(byte[] b1, int s1, int l1,
- byte[] b2, int s2, int l2) {
-
- if (sortSpec == null) {
- return super.compare(b1, s1, l1, b2, s2, l2);
- }
-
- try {
- Text logline1 = new Text();
- logline1.readFields(new DataInputStream(new ByteArrayInputStream(b1, s1, l1)));
- String line1 = logline1.toString();
- String[] logColumns1 = line1.split(columnSeparator);
-
- Text logline2 = new Text();
- logline2.readFields(new DataInputStream(new ByteArrayInputStream(b2, s2, l2)));
- String line2 = logline2.toString();
- String[] logColumns2 = line2.split(columnSeparator);
-
- if (logColumns1 == null || logColumns2 == null) {
- return super.compare(b1, s1, l1, b2, s2, l2);
- }
-
- //Compare column-wise according to *sortSpec*
- for(int i=0; i < sortSpec.length; ++i) {
- int column = Integer.parseInt(sortSpec[i]);
- String c1 = logColumns1[column];
- String c2 = logColumns2[column];
-
- //Compare columns
- int comparision = super.compareBytes(
- c1.getBytes(Charset.forName("UTF-8")), 0, c1.length(),
- c2.getBytes(Charset.forName("UTF-8")), 0, c2.length()
- );
-
- //They differ!
- if (comparision != 0) {
- return comparision;
- }
- }
-
- } catch (IOException ioe) {
- LOG.fatal("Caught " + ioe);
- return 0;
- }
-
- return 0;
- }
-
- static {
- // register this comparator
- WritableComparator.define(Text.class, new LogComparator());
- }
- }
-
- /**
- * doArchive: Workhorse function to archive log-files.
- * @param logListURI : The uri which will serve list of log-files to archive.
- * @param archiveDirectory : The directory to store archived logfiles.
- * @throws IOException
- */
- @SuppressWarnings("deprecation")
- public void
- doArchive(String logListURI, String archiveDirectory)
- throws IOException
- {
- String destURL = FileSystem.getDefaultUri(fsConfig) + archiveDirectory;
- DistCpV1.copy(new JobConf(fsConfig), logListURI, destURL, null, true, false);
- }
-
- /**
- * doAnalyze:
- * @param inputFilesDirectory : Directory containing the files to be analyzed.
- * @param outputDirectory : Directory to store analysis (output).
- * @param grepPattern : Pattern to *grep* for.
- * @param sortColumns : Sort specification for output.
- * @param columnSeparator : Column separator.
- * @throws IOException
- */
- public void
- doAnalyze(String inputFilesDirectory, String outputDirectory,
- String grepPattern, String sortColumns, String columnSeparator)
- throws IOException
- {
- Path grepInput = new Path(inputFilesDirectory);
-
- Path analysisOutput = null;
- if (outputDirectory.equals("")) {
- analysisOutput = new Path(inputFilesDirectory, "logalyzer_" +
- Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
- } else {
- analysisOutput = new Path(outputDirectory);
- }
-
- JobConf grepJob = new JobConf(fsConfig);
- grepJob.setJobName("logalyzer-grep-sort");
-
- FileInputFormat.setInputPaths(grepJob, grepInput);
- grepJob.setInputFormat(TextInputFormat.class);
-
- grepJob.setMapperClass(LogRegexMapper.class);
- grepJob.set(RegexMapper.PATTERN, grepPattern);
- grepJob.set(SORT_COLUMNS, sortColumns);
- grepJob.set(COLUMN_SEPARATOR, columnSeparator);
-
- grepJob.setCombinerClass(LongSumReducer.class);
- grepJob.setReducerClass(LongSumReducer.class);
-
- FileOutputFormat.setOutputPath(grepJob, analysisOutput);
- grepJob.setOutputFormat(TextOutputFormat.class);
- grepJob.setOutputKeyClass(Text.class);
- grepJob.setOutputValueClass(LongWritable.class);
- grepJob.setOutputKeyComparatorClass(LogComparator.class);
-
- grepJob.setNumReduceTasks(1); // write a single file
-
- JobClient.runJob(grepJob);
- }
-
- public static void main(String[] args) {
-
- Log LOG = LogFactory.getLog(Logalyzer.class);
-
- String version = "Logalyzer.0.0.1";
- String usage = "Usage: Logalyzer [-archive -logs <urlsFile>] " +
- "-archiveDir <archiveDirectory> " +
- "-grep <pattern> -sort <column1,column2,...> -separator <separator> " +
- "-analysis <outputDirectory>";
-
- System.out.println(version);
- if (args.length == 0) {
- System.err.println(usage);
- System.exit(-1);
- }
-
- //Command line arguments
- boolean archive = false;
- boolean grep = false;
- boolean sort = false;
-
- String archiveDir = "";
- String logListURI = "";
- String grepPattern = ".*";
- String sortColumns = "";
- String columnSeparator = " ";
- String outputDirectory = "";
-
- for (int i = 0; i < args.length; i++) { // parse command line
- if (args[i].equals("-archive")) {
- archive = true;
- } else if (args[i].equals("-archiveDir")) {
- archiveDir = args[++i];
- } else if (args[i].equals("-grep")) {
- grep = true;
- grepPattern = args[++i];
- } else if (args[i].equals("-logs")) {
- logListURI = args[++i];
- } else if (args[i].equals("-sort")) {
- sort = true;
- sortColumns = args[++i];
- } else if (args[i].equals("-separator")) {
- columnSeparator = args[++i];
- } else if (args[i].equals("-analysis")) {
- outputDirectory = args[++i];
- }
- }
-
- LOG.info("analysisDir = " + outputDirectory);
- LOG.info("archiveDir = " + archiveDir);
- LOG.info("logListURI = " + logListURI);
- LOG.info("grepPattern = " + grepPattern);
- LOG.info("sortColumns = " + sortColumns);
- LOG.info("separator = " + columnSeparator);
-
- try {
- Logalyzer logalyzer = new Logalyzer();
-
- // Archive?
- if (archive) {
- logalyzer.doArchive(logListURI, archiveDir);
- }
-
- // Analyze?
- if (grep || sort) {
- logalyzer.doAnalyze(archiveDir, outputDirectory, grepPattern, sortColumns, columnSeparator);
- }
- } catch (IOException ioe) {
- ioe.printStackTrace();
- System.exit(-1);
- }
-
- } //main
-
-} //class Logalyzer
[10/50] [abbrv] hadoop git commit: HADOOP-11944. add option to
test-patch to avoid relocating patch process directory (Sean Busbey via aw)
Posted by zj...@apache.org.
HADOOP-11944. add option to test-patch to avoid relocating patch process directory (Sean Busbey via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f120e135
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f120e135
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f120e135
Branch: refs/heads/YARN-2928
Commit: f120e135f7850f3bfe914091ae3d88d6afd2164e
Parents: bb44e04
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon May 18 16:13:50 2015 +0000
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:47:02 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.sh | 28 +++++++++++---------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
2 files changed, 18 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f120e135/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 00a638c..ae74c5b 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -38,6 +38,7 @@ function setup_defaults
HOW_TO_CONTRIBUTE="https://wiki.apache.org/hadoop/HowToContribute"
JENKINS=false
BASEDIR=$(pwd)
+ RELOCATE_PATCH_DIR=false
FINDBUGS_HOME=${FINDBUGS_HOME:-}
ECLIPSE_HOME=${ECLIPSE_HOME:-}
@@ -607,6 +608,7 @@ function hadoop_usage
echo "--eclipse-home=<path> Eclipse home directory (default ECLIPSE_HOME environment variable)"
echo "--jira-cmd=<cmd> The 'jira' command to use (default 'jira')"
echo "--jira-password=<pw> The password for the 'jira' command"
+ echo "--mv-patch-dir Move the patch-dir into the basedir during cleanup."
echo "--wget-cmd=<cmd> The 'wget' command to use (default 'wget')"
}
@@ -692,6 +694,9 @@ function parse_args
--mvn-cmd=*)
MVN=${i#*=}
;;
+ --mv-patch-dir)
+ RELOCATE_PATCH_DIR=true;
+ ;;
--offline)
OFFLINE=true
;;
@@ -2323,19 +2328,16 @@ function cleanup_and_exit
{
local result=$1
- if [[ ${JENKINS} == "true" ]] ; then
- if [[ -e "${PATCH_DIR}" ]] ; then
- if [[ -d "${PATCH_DIR}" ]]; then
- # if PATCH_DIR is already inside BASEDIR, then
- # there is no need to move it since we assume that
- # Jenkins or whatever already knows where it is at
- # since it told us to put it there!
- relative_patchdir >/dev/null
- if [[ $? == 1 ]]; then
- hadoop_debug "mv ${PATCH_DIR} ${BASEDIR}"
- mv "${PATCH_DIR}" "${BASEDIR}"
- fi
- fi
+ if [[ ${JENKINS} == "true" && ${RELOCATE_PATCH_DIR} == "true" && \
+ -e ${PATCH_DIR} && -d ${PATCH_DIR} ]] ; then
+ # if PATCH_DIR is already inside BASEDIR, then
+ # there is no need to move it since we assume that
+ # Jenkins or whatever already knows where it is at
+ # since it told us to put it there!
+ relative_patchdir >/dev/null
+ if [[ $? == 1 ]]; then
+ hadoop_debug "mv ${PATCH_DIR} ${BASEDIR}"
+ mv "${PATCH_DIR}" "${BASEDIR}"
fi
fi
big_console_header "Finished build."
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f120e135/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 1c2cdaa..8f66072 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -578,6 +578,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11884. test-patch.sh should pull the real findbugs version
(Kengo Seki via aw)
+ HADOOP-11944. add option to test-patch to avoid relocating patch process
+ directory (Sean Busbey via aw)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
[49/50] [abbrv] hadoop git commit: HADOOP-10366. Add whitespaces
between classes for values in core-default.xml to fit better in browser.
Contributed by kanaka kumar avvaru.
Posted by zj...@apache.org.
HADOOP-10366. Add whitespaces between classes for values in core-default.xml to fit better in browser. Contributed by kanaka kumar avvaru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f2297c9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f2297c9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f2297c9
Branch: refs/heads/YARN-2928
Commit: 5f2297c975b1e6a5be09b047dcea1579968e7de5
Parents: e6cc8c4
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu May 21 17:52:03 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 12:42:58 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 4 ++++
.../hadoop-common/src/main/resources/core-default.xml | 4 ++--
.../hadoop-hdfs/src/site/markdown/TransparentEncryption.md | 2 +-
3 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f2297c9/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 416b819..aff9368 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -593,6 +593,10 @@ Release 2.8.0 - UNRELEASED
HADOOP-11995. Make jetty version configurable from the maven command line.
(Sriharsha Devineni via wheat9)
+ HADOOP-10366. Add whitespaces between classes for values in
+ core-default.xml to fit better in browser.
+ (kanaka kumar avvaru via aajisaka)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f2297c9/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 97e01a8..a1bc780 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -466,7 +466,7 @@ for ldap providers in the same way as above does.
<property>
<name>io.serializations</name>
- <value>org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization</value>
+ <value>org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization</value>
<description>A list of serialization classes that can be used for
obtaining serializers and deserializers.</description>
</property>
@@ -1655,7 +1655,7 @@ for ldap providers in the same way as above does.
<property>
<name>hadoop.security.crypto.codec.classes.aes.ctr.nopadding</name>
- <value>org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec,org.apache.hadoop.crypto.JceAesCtrCryptoCodec</value>
+ <value>org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec</value>
<description>
Comma-separated list of crypto codec implementations for AES/CTR/NoPadding.
The first implementation will be used if available, others are fallbacks.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f2297c9/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
index aa2acbd..05e4249 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
@@ -126,7 +126,7 @@ The prefix for a given crypto codec, contains a comma-separated list of implemen
#### hadoop.security.crypto.codec.classes.aes.ctr.nopadding
-Default: `org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec,org.apache.hadoop.crypto.JceAesCtrCryptoCodec`
+Default: `org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec`
Comma-separated list of crypto codec implementations for AES/CTR/NoPadding. The first implementation will be used if available, others are fallbacks.
[36/50] [abbrv] hadoop git commit: YARN-3583. Support of NodeLabel
object instead of plain String in YarnClient side. (Sunil G via wangda)
Posted by zj...@apache.org.
YARN-3583. Support of NodeLabel object instead of plain String in YarnClient side. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7ff5819
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7ff5819
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7ff5819
Branch: refs/heads/YARN-2928
Commit: c7ff58191fc42bb73a32551d380a45aa0d329b1b
Parents: 69e6c7b
Author: Wangda Tan <wa...@apache.org>
Authored: Tue May 19 16:54:38 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 12:42:56 2015 -0700
----------------------------------------------------------------------
.../hadoop/mapred/ResourceMgrDelegate.java | 6 +-
hadoop-yarn-project/CHANGES.txt | 3 +
.../GetLabelsToNodesResponse.java | 7 +-
.../GetNodesToLabelsResponse.java | 7 +-
..._server_resourcemanager_service_protos.proto | 7 +-
.../src/main/proto/yarn_protos.proto | 6 +-
.../src/main/proto/yarn_service_protos.proto | 2 +-
.../hadoop/yarn/client/api/YarnClient.java | 8 +-
.../yarn/client/api/impl/YarnClientImpl.java | 6 +-
.../yarn/client/api/impl/TestYarnClient.java | 74 +++++++++++--
.../impl/pb/GetLabelsToNodesResponsePBImpl.java | 28 +++--
.../impl/pb/GetNodesToLabelsResponsePBImpl.java | 58 ++++++----
.../pb/ReplaceLabelsOnNodeRequestPBImpl.java | 18 +--
.../server/resourcemanager/ClientRMService.java | 6 +-
.../resourcemanager/TestClientRMService.java | 110 +++++++++++--------
15 files changed, 226 insertions(+), 120 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ff5819/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
index 2b7cd5f..90f6876 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
@@ -444,19 +444,19 @@ public class ResourceMgrDelegate extends YarnClient {
}
@Override
- public Map<NodeId, Set<String>> getNodeToLabels() throws YarnException,
+ public Map<NodeId, Set<NodeLabel>> getNodeToLabels() throws YarnException,
IOException {
return client.getNodeToLabels();
}
@Override
- public Map<String, Set<NodeId>> getLabelsToNodes() throws YarnException,
+ public Map<NodeLabel, Set<NodeId>> getLabelsToNodes() throws YarnException,
IOException {
return client.getLabelsToNodes();
}
@Override
- public Map<String, Set<NodeId>> getLabelsToNodes(Set<String> labels)
+ public Map<NodeLabel, Set<NodeId>> getLabelsToNodes(Set<String> labels)
throws YarnException, IOException {
return client.getLabelsToNodes(labels);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ff5819/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b6b6c9c..46a0e51 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -331,6 +331,9 @@ Release 2.8.0 - UNRELEASED
YARN-3565. NodeHeartbeatRequest/RegisterNodeManagerRequest should use
NodeLabel object instead of String. (Naganarasimha G R via wangda)
+ YARN-3583. Support of NodeLabel object instead of plain String
+ in YarnClient side. (Sunil G via wangda)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ff5819/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java
index f105359..da2be28 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLabelsToNodesResponse.java
@@ -24,11 +24,12 @@ import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.util.Records;
public abstract class GetLabelsToNodesResponse {
public static GetLabelsToNodesResponse newInstance(
- Map<String, Set<NodeId>> map) {
+ Map<NodeLabel, Set<NodeId>> map) {
GetLabelsToNodesResponse response =
Records.newRecord(GetLabelsToNodesResponse.class);
response.setLabelsToNodes(map);
@@ -37,9 +38,9 @@ public abstract class GetLabelsToNodesResponse {
@Public
@Evolving
- public abstract void setLabelsToNodes(Map<String, Set<NodeId>> map);
+ public abstract void setLabelsToNodes(Map<NodeLabel, Set<NodeId>> map);
@Public
@Evolving
- public abstract Map<String, Set<NodeId>> getLabelsToNodes();
+ public abstract Map<NodeLabel, Set<NodeId>> getLabelsToNodes();
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ff5819/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java
index bcd5421..432485c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetNodesToLabelsResponse.java
@@ -24,11 +24,12 @@ import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.util.Records;
public abstract class GetNodesToLabelsResponse {
public static GetNodesToLabelsResponse newInstance(
- Map<NodeId, Set<String>> map) {
+ Map<NodeId, Set<NodeLabel>> map) {
GetNodesToLabelsResponse response =
Records.newRecord(GetNodesToLabelsResponse.class);
response.setNodeToLabels(map);
@@ -37,9 +38,9 @@ public abstract class GetNodesToLabelsResponse {
@Public
@Evolving
- public abstract void setNodeToLabels(Map<NodeId, Set<String>> map);
+ public abstract void setNodeToLabels(Map<NodeId, Set<NodeLabel>> map);
@Public
@Evolving
- public abstract Map<NodeId, Set<String>> getNodeToLabels();
+ public abstract Map<NodeId, Set<NodeLabel>> getNodeToLabels();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ff5819/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
index d6d8713..e20b4ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
@@ -91,7 +91,7 @@ message RemoveFromClusterNodeLabelsResponseProto {
}
message ReplaceLabelsOnNodeRequestProto {
- repeated NodeIdToLabelsProto nodeToLabels = 1;
+ repeated NodeIdToLabelsNameProto nodeToLabels = 1;
}
message ReplaceLabelsOnNodeResponseProto {
@@ -107,6 +107,11 @@ message CheckForDecommissioningNodesResponseProto {
repeated NodeIdProto decommissioningNodes = 1;
}
+message NodeIdToLabelsNameProto {
+ optional NodeIdProto nodeId = 1;
+ repeated string nodeLabels = 2;
+}
+
enum DecommissionTypeProto {
NORMAL = 1;
GRACEFUL = 2;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ff5819/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 3c4aa52..b9969b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -248,13 +248,13 @@ message NodeReportProto {
repeated string node_labels = 10;
}
-message NodeIdToLabelsProto {
+message NodeIdToLabelsInfoProto {
optional NodeIdProto nodeId = 1;
- repeated string nodeLabels = 2;
+ repeated NodeLabelProto nodeLabels = 2;
}
message LabelsToNodeIdsProto {
- optional string nodeLabels = 1;
+ optional NodeLabelProto nodeLabels = 1;
repeated NodeIdProto nodeId = 2;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ff5819/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index c02e1c9..94564c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -199,7 +199,7 @@ message GetNodesToLabelsRequestProto {
}
message GetNodesToLabelsResponseProto {
- repeated NodeIdToLabelsProto nodeToLabels = 1;
+ repeated NodeIdToLabelsInfoProto nodeToLabels = 1;
}
message GetLabelsToNodesRequestProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ff5819/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
index 5ce626c..ff03c7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
@@ -619,7 +619,7 @@ public abstract class YarnClient extends AbstractService {
*/
@Public
@Unstable
- public abstract Map<NodeId, Set<String>> getNodeToLabels()
+ public abstract Map<NodeId, Set<NodeLabel>> getNodeToLabels()
throws YarnException, IOException;
/**
@@ -634,7 +634,7 @@ public abstract class YarnClient extends AbstractService {
*/
@Public
@Unstable
- public abstract Map<String, Set<NodeId>> getLabelsToNodes()
+ public abstract Map<NodeLabel, Set<NodeId>> getLabelsToNodes()
throws YarnException, IOException;
/**
@@ -650,8 +650,8 @@ public abstract class YarnClient extends AbstractService {
*/
@Public
@Unstable
- public abstract Map<String, Set<NodeId>> getLabelsToNodes(Set<String> labels)
- throws YarnException, IOException;
+ public abstract Map<NodeLabel, Set<NodeId>> getLabelsToNodes(
+ Set<String> labels) throws YarnException, IOException;
/**
* <p>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ff5819/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index 42dd5cd..be4c8c4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -795,21 +795,21 @@ public class YarnClientImpl extends YarnClient {
}
@Override
- public Map<NodeId, Set<String>> getNodeToLabels() throws YarnException,
+ public Map<NodeId, Set<NodeLabel>> getNodeToLabels() throws YarnException,
IOException {
return rmClient.getNodeToLabels(GetNodesToLabelsRequest.newInstance())
.getNodeToLabels();
}
@Override
- public Map<String, Set<NodeId>> getLabelsToNodes() throws YarnException,
+ public Map<NodeLabel, Set<NodeId>> getLabelsToNodes() throws YarnException,
IOException {
return rmClient.getLabelsToNodes(GetLabelsToNodesRequest.newInstance())
.getLabelsToNodes();
}
@Override
- public Map<String, Set<NodeId>> getLabelsToNodes(Set<String> labels)
+ public Map<NodeLabel, Set<NodeId>> getLabelsToNodes(Set<String> labels)
throws YarnException, IOException {
return rmClient.getLabelsToNodes(
GetLabelsToNodesRequest.newInstance(labels)).getLabelsToNodes();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ff5819/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index 10b9bbb..511fa4a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -67,6 +67,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest;
@@ -87,6 +89,7 @@ import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
import org.apache.hadoop.yarn.api.records.ReservationId;
@@ -458,9 +461,9 @@ public class TestYarnClient {
client.start();
// Get labels to nodes mapping
- Map<String, Set<NodeId>> expectedLabelsToNodes =
+ Map<NodeLabel, Set<NodeId>> expectedLabelsToNodes =
((MockYarnClient)client).getLabelsToNodesMap();
- Map<String, Set<NodeId>> labelsToNodes = client.getLabelsToNodes();
+ Map<NodeLabel, Set<NodeId>> labelsToNodes = client.getLabelsToNodes();
Assert.assertEquals(labelsToNodes, expectedLabelsToNodes);
Assert.assertEquals(labelsToNodes.size(), 3);
@@ -476,7 +479,32 @@ public class TestYarnClient {
client.close();
}
+ @Test (timeout = 10000)
+ public void testGetNodesToLabels() throws YarnException, IOException {
+ Configuration conf = new Configuration();
+ final YarnClient client = new MockYarnClient();
+ client.init(conf);
+ client.start();
+
+ // Get labels to nodes mapping
+ Map<NodeId, Set<NodeLabel>> expectedNodesToLabels = ((MockYarnClient) client)
+ .getNodeToLabelsMap();
+ Map<NodeId, Set<NodeLabel>> nodesToLabels = client.getNodeToLabels();
+ Assert.assertEquals(nodesToLabels, expectedNodesToLabels);
+ Assert.assertEquals(nodesToLabels.size(), 1);
+
+ // Verify exclusivity
+ Set<NodeLabel> labels = nodesToLabels.get(NodeId.newInstance("host", 0));
+ for (NodeLabel label : labels) {
+ Assert.assertFalse(label.isExclusive());
+ }
+
+ client.stop();
+ client.close();
+ }
+
private static class MockYarnClient extends YarnClientImpl {
+
private ApplicationReport mockReport;
private List<ApplicationReport> reports;
private HashMap<ApplicationId, List<ApplicationAttemptReport>> attempts =
@@ -498,6 +526,8 @@ public class TestYarnClient {
mock(GetContainerReportResponse.class);
GetLabelsToNodesResponse mockLabelsToNodesResponse =
mock(GetLabelsToNodesResponse.class);
+ GetNodesToLabelsResponse mockNodeToLabelsResponse =
+ mock(GetNodesToLabelsResponse.class);
public MockYarnClient() {
super();
@@ -537,6 +567,9 @@ public class TestYarnClient {
when(rmClient.getLabelsToNodes(any(GetLabelsToNodesRequest.class)))
.thenReturn(mockLabelsToNodesResponse);
+ when(rmClient.getNodeToLabels(any(GetNodesToLabelsRequest.class)))
+ .thenReturn(mockNodeToLabelsResponse);
+
historyClient = mock(AHSClient.class);
} catch (YarnException e) {
@@ -704,7 +737,7 @@ public class TestYarnClient {
}
@Override
- public Map<String, Set<NodeId>> getLabelsToNodes()
+ public Map<NodeLabel, Set<NodeId>> getLabelsToNodes()
throws YarnException, IOException {
when(mockLabelsToNodesResponse.getLabelsToNodes()).thenReturn(
getLabelsToNodesMap());
@@ -712,36 +745,53 @@ public class TestYarnClient {
}
@Override
- public Map<String, Set<NodeId>> getLabelsToNodes(Set<String> labels)
+ public Map<NodeLabel, Set<NodeId>> getLabelsToNodes(Set<String> labels)
throws YarnException, IOException {
when(mockLabelsToNodesResponse.getLabelsToNodes()).thenReturn(
getLabelsToNodesMap(labels));
return super.getLabelsToNodes(labels);
}
- public Map<String, Set<NodeId>> getLabelsToNodesMap() {
- Map<String, Set<NodeId>> map = new HashMap<String, Set<NodeId>>();
+ public Map<NodeLabel, Set<NodeId>> getLabelsToNodesMap() {
+ Map<NodeLabel, Set<NodeId>> map = new HashMap<NodeLabel, Set<NodeId>>();
Set<NodeId> setNodeIds =
new HashSet<NodeId>(Arrays.asList(
NodeId.newInstance("host1", 0), NodeId.newInstance("host2", 0)));
- map.put("x", setNodeIds);
- map.put("y", setNodeIds);
- map.put("z", setNodeIds);
+ map.put(NodeLabel.newInstance("x"), setNodeIds);
+ map.put(NodeLabel.newInstance("y"), setNodeIds);
+ map.put(NodeLabel.newInstance("z"), setNodeIds);
return map;
}
- public Map<String, Set<NodeId>> getLabelsToNodesMap(Set<String> labels) {
- Map<String, Set<NodeId>> map = new HashMap<String, Set<NodeId>>();
+ public Map<NodeLabel, Set<NodeId>> getLabelsToNodesMap(Set<String> labels) {
+ Map<NodeLabel, Set<NodeId>> map = new HashMap<NodeLabel, Set<NodeId>>();
Set<NodeId> setNodeIds =
new HashSet<NodeId>(Arrays.asList(
NodeId.newInstance("host1", 0), NodeId.newInstance("host2", 0)));
for(String label : labels) {
- map.put(label, setNodeIds);
+ map.put(NodeLabel.newInstance(label), setNodeIds);
}
return map;
}
@Override
+ public Map<NodeId, Set<NodeLabel>> getNodeToLabels() throws YarnException,
+ IOException {
+ when(mockNodeToLabelsResponse.getNodeToLabels()).thenReturn(
+ getNodeToLabelsMap());
+ return super.getNodeToLabels();
+ }
+
+ public Map<NodeId, Set<NodeLabel>> getNodeToLabelsMap() {
+ Map<NodeId, Set<NodeLabel>> map = new HashMap<NodeId, Set<NodeLabel>>();
+ Set<NodeLabel> setNodeLabels = new HashSet<NodeLabel>(Arrays.asList(
+ NodeLabel.newInstance("x", false),
+ NodeLabel.newInstance("y", false)));
+ map.put(NodeId.newInstance("host", 0), setNodeLabels);
+ return map;
+ }
+
+ @Override
public List<ApplicationAttemptReport> getApplicationAttempts(
ApplicationId appId) throws YarnException, IOException {
when(mockAppAttemptsResponse.getApplicationAttemptList()).thenReturn(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ff5819/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java
index e197997..418fcbd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLabelsToNodesResponsePBImpl.java
@@ -29,11 +29,13 @@ import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse;
-
import org.apache.hadoop.yarn.proto.YarnProtos.LabelsToNodeIdsProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetLabelsToNodesResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetLabelsToNodesResponseProtoOrBuilder;
@@ -44,7 +46,7 @@ public class GetLabelsToNodesResponsePBImpl extends
GetLabelsToNodesResponseProto.Builder builder = null;
boolean viaProto = false;
- private Map<String, Set<NodeId>> labelsToNodes;
+ private Map<NodeLabel, Set<NodeId>> labelsToNodes;
public GetLabelsToNodesResponsePBImpl() {
this.builder = GetLabelsToNodesResponseProto.newBuilder();
@@ -61,7 +63,7 @@ public class GetLabelsToNodesResponsePBImpl extends
}
GetLabelsToNodesResponseProtoOrBuilder p = viaProto ? proto : builder;
List<LabelsToNodeIdsProto> list = p.getLabelsToNodesList();
- this.labelsToNodes = new HashMap<String, Set<NodeId>>();
+ this.labelsToNodes = new HashMap<NodeLabel, Set<NodeId>>();
for (LabelsToNodeIdsProto c : list) {
Set<NodeId> setNodes = new HashSet<NodeId>();
@@ -69,8 +71,9 @@ public class GetLabelsToNodesResponsePBImpl extends
NodeId node = new NodeIdPBImpl(n);
setNodes.add(node);
}
- if(!setNodes.isEmpty()) {
- this.labelsToNodes.put(c.getNodeLabels(), setNodes);
+ if (!setNodes.isEmpty()) {
+ this.labelsToNodes
+ .put(new NodeLabelPBImpl(c.getNodeLabels()), setNodes);
}
}
}
@@ -94,7 +97,7 @@ public class GetLabelsToNodesResponsePBImpl extends
public Iterator<LabelsToNodeIdsProto> iterator() {
return new Iterator<LabelsToNodeIdsProto>() {
- Iterator<Entry<String, Set<NodeId>>> iter =
+ Iterator<Entry<NodeLabel, Set<NodeId>>> iter =
labelsToNodes.entrySet().iterator();
@Override
@@ -104,13 +107,14 @@ public class GetLabelsToNodesResponsePBImpl extends
@Override
public LabelsToNodeIdsProto next() {
- Entry<String, Set<NodeId>> now = iter.next();
+ Entry<NodeLabel, Set<NodeId>> now = iter.next();
Set<NodeIdProto> nodeProtoSet = new HashSet<NodeIdProto>();
for(NodeId n : now.getValue()) {
nodeProtoSet.add(convertToProtoFormat(n));
}
return LabelsToNodeIdsProto.newBuilder()
- .setNodeLabels(now.getKey()).addAllNodeId(nodeProtoSet)
+ .setNodeLabels(convertToProtoFormat(now.getKey()))
+ .addAllNodeId(nodeProtoSet)
.build();
}
@@ -149,6 +153,10 @@ public class GetLabelsToNodesResponsePBImpl extends
return ((NodeIdPBImpl)t).getProto();
}
+ private NodeLabelProto convertToProtoFormat(NodeLabel l) {
+ return ((NodeLabelPBImpl)l).getProto();
+ }
+
@Override
public int hashCode() {
assert false : "hashCode not designed";
@@ -168,7 +176,7 @@ public class GetLabelsToNodesResponsePBImpl extends
@Override
@Public
@Evolving
- public void setLabelsToNodes(Map<String, Set<NodeId>> map) {
+ public void setLabelsToNodes(Map<NodeLabel, Set<NodeId>> map) {
initLabelsToNodes();
labelsToNodes.clear();
labelsToNodes.putAll(map);
@@ -177,7 +185,7 @@ public class GetLabelsToNodesResponsePBImpl extends
@Override
@Public
@Evolving
- public Map<String, Set<NodeId>> getLabelsToNodes() {
+ public Map<NodeLabel, Set<NodeId>> getLabelsToNodes() {
initLabelsToNodes();
return this.labelsToNodes;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ff5819/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java
index 3404830..52be73f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetNodesToLabelsResponsePBImpl.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -26,12 +27,13 @@ import java.util.Map.Entry;
import java.util.Set;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse;
-
-import com.google.common.collect.Sets;
-import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdToLabelsProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdToLabelsInfoProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNodesToLabelsResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetNodesToLabelsResponseProtoOrBuilder;
@@ -42,8 +44,8 @@ public class GetNodesToLabelsResponsePBImpl extends
GetNodesToLabelsResponseProto.Builder builder = null;
boolean viaProto = false;
- private Map<NodeId, Set<String>> nodeToLabels;
-
+ private Map<NodeId, Set<NodeLabel>> nodeToLabels;
+
public GetNodesToLabelsResponsePBImpl() {
this.builder = GetNodesToLabelsResponseProto.newBuilder();
}
@@ -58,12 +60,15 @@ public class GetNodesToLabelsResponsePBImpl extends
return;
}
GetNodesToLabelsResponseProtoOrBuilder p = viaProto ? proto : builder;
- List<NodeIdToLabelsProto> list = p.getNodeToLabelsList();
- this.nodeToLabels = new HashMap<NodeId, Set<String>>();
-
- for (NodeIdToLabelsProto c : list) {
- this.nodeToLabels.put(new NodeIdPBImpl(c.getNodeId()),
- Sets.newHashSet(c.getNodeLabelsList()));
+ List<NodeIdToLabelsInfoProto> list = p.getNodeToLabelsList();
+ this.nodeToLabels = new HashMap<NodeId, Set<NodeLabel>>();
+
+ for (NodeIdToLabelsInfoProto c : list) {
+ Set<NodeLabel> labels = new HashSet<NodeLabel>();
+ for (NodeLabelProto l : c.getNodeLabelsList()) {
+ labels.add(new NodeLabelPBImpl(l));
+ }
+ this.nodeToLabels.put(new NodeIdPBImpl(c.getNodeId()), labels);
}
}
@@ -80,13 +85,13 @@ public class GetNodesToLabelsResponsePBImpl extends
if (nodeToLabels == null) {
return;
}
- Iterable<NodeIdToLabelsProto> iterable =
- new Iterable<NodeIdToLabelsProto>() {
+ Iterable<NodeIdToLabelsInfoProto> iterable =
+ new Iterable<NodeIdToLabelsInfoProto>() {
@Override
- public Iterator<NodeIdToLabelsProto> iterator() {
- return new Iterator<NodeIdToLabelsProto>() {
+ public Iterator<NodeIdToLabelsInfoProto> iterator() {
+ return new Iterator<NodeIdToLabelsInfoProto>() {
- Iterator<Entry<NodeId, Set<String>>> iter = nodeToLabels
+ Iterator<Entry<NodeId, Set<NodeLabel>>> iter = nodeToLabels
.entrySet().iterator();
@Override
@@ -95,11 +100,16 @@ public class GetNodesToLabelsResponsePBImpl extends
}
@Override
- public NodeIdToLabelsProto next() {
- Entry<NodeId, Set<String>> now = iter.next();
- return NodeIdToLabelsProto.newBuilder()
+ public NodeIdToLabelsInfoProto next() {
+ Entry<NodeId, Set<NodeLabel>> now = iter.next();
+ Set<NodeLabelProto> labelProtoList =
+ new HashSet<NodeLabelProto>();
+ for (NodeLabel l : now.getValue()) {
+ labelProtoList.add(convertToProtoFormat(l));
+ }
+ return NodeIdToLabelsInfoProto.newBuilder()
.setNodeId(convertToProtoFormat(now.getKey()))
- .addAllNodeLabels(now.getValue()).build();
+ .addAllNodeLabels(labelProtoList).build();
}
@Override
@@ -134,13 +144,13 @@ public class GetNodesToLabelsResponsePBImpl extends
}
@Override
- public Map<NodeId, Set<String>> getNodeToLabels() {
+ public Map<NodeId, Set<NodeLabel>> getNodeToLabels() {
initNodeToLabels();
return this.nodeToLabels;
}
@Override
- public void setNodeToLabels(Map<NodeId, Set<String>> map) {
+ public void setNodeToLabels(Map<NodeId, Set<NodeLabel>> map) {
initNodeToLabels();
nodeToLabels.clear();
nodeToLabels.putAll(map);
@@ -150,6 +160,10 @@ public class GetNodesToLabelsResponsePBImpl extends
return ((NodeIdPBImpl)t).getProto();
}
+ private NodeLabelProto convertToProtoFormat(NodeLabel t) {
+ return ((NodeLabelPBImpl)t).getProto();
+ }
+
@Override
public int hashCode() {
assert false : "hashCode not designed";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ff5819/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java
index e296aaf..22e561c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReplaceLabelsOnNodeRequestPBImpl.java
@@ -28,7 +28,7 @@ import java.util.Set;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdToLabelsProto;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodeIdToLabelsNameProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProtoOrBuilder;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest;
@@ -58,10 +58,10 @@ public class ReplaceLabelsOnNodeRequestPBImpl extends
return;
}
ReplaceLabelsOnNodeRequestProtoOrBuilder p = viaProto ? proto : builder;
- List<NodeIdToLabelsProto> list = p.getNodeToLabelsList();
+ List<NodeIdToLabelsNameProto> list = p.getNodeToLabelsList();
this.nodeIdToLabels = new HashMap<NodeId, Set<String>>();
- for (NodeIdToLabelsProto c : list) {
+ for (NodeIdToLabelsNameProto c : list) {
this.nodeIdToLabels.put(new NodeIdPBImpl(c.getNodeId()),
Sets.newHashSet(c.getNodeLabelsList()));
}
@@ -80,11 +80,11 @@ public class ReplaceLabelsOnNodeRequestPBImpl extends
if (nodeIdToLabels == null) {
return;
}
- Iterable<NodeIdToLabelsProto> iterable =
- new Iterable<NodeIdToLabelsProto>() {
+ Iterable<NodeIdToLabelsNameProto> iterable =
+ new Iterable<NodeIdToLabelsNameProto>() {
@Override
- public Iterator<NodeIdToLabelsProto> iterator() {
- return new Iterator<NodeIdToLabelsProto>() {
+ public Iterator<NodeIdToLabelsNameProto> iterator() {
+ return new Iterator<NodeIdToLabelsNameProto>() {
Iterator<Entry<NodeId, Set<String>>> iter = nodeIdToLabels
.entrySet().iterator();
@@ -95,9 +95,9 @@ public class ReplaceLabelsOnNodeRequestPBImpl extends
}
@Override
- public NodeIdToLabelsProto next() {
+ public NodeIdToLabelsNameProto next() {
Entry<NodeId, Set<String>> now = iter.next();
- return NodeIdToLabelsProto.newBuilder()
+ return NodeIdToLabelsNameProto.newBuilder()
.setNodeId(convertToProtoFormat(now.getKey())).clearNodeLabels()
.addAllNodeLabels(now.getValue()).build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ff5819/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index a5e6789..c7d3da2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -1248,7 +1248,7 @@ public class ClientRMService extends AbstractService implements
GetNodesToLabelsRequest request) throws YarnException, IOException {
RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager();
GetNodesToLabelsResponse response =
- GetNodesToLabelsResponse.newInstance(labelsMgr.getNodeLabels());
+ GetNodesToLabelsResponse.newInstance(labelsMgr.getNodeLabelsInfo());
return response;
}
@@ -1258,10 +1258,10 @@ public class ClientRMService extends AbstractService implements
RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager();
if (request.getNodeLabels() == null || request.getNodeLabels().isEmpty()) {
return GetLabelsToNodesResponse.newInstance(
- labelsMgr.getLabelsToNodes());
+ labelsMgr.getLabelsInfoToNodes());
} else {
return GetLabelsToNodesResponse.newInstance(
- labelsMgr.getLabelsToNodes(request.getNodeLabels()));
+ labelsMgr.getLabelsInfoToNodes(request.getNodeLabels()));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ff5819/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 85f8b9f..8eac6f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -1417,8 +1417,10 @@ public class TestClientRMService {
};
};
rm.start();
+ NodeLabel labelX = NodeLabel.newInstance("x", false);
+ NodeLabel labelY = NodeLabel.newInstance("y");
RMNodeLabelsManager labelsMgr = rm.getRMContext().getNodeLabelManager();
- labelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y"));
+ labelsMgr.addToCluserNodeLabels(ImmutableSet.of(labelX, labelY));
NodeId node1 = NodeId.newInstance("host1", 1234);
NodeId node2 = NodeId.newInstance("host2", 1234);
@@ -1432,25 +1434,37 @@ public class TestClientRMService {
YarnRPC rpc = YarnRPC.create(conf);
InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
LOG.info("Connecting to ResourceManager at " + rmAddress);
- ApplicationClientProtocol client =
- (ApplicationClientProtocol) rpc.getProxy(
- ApplicationClientProtocol.class, rmAddress, conf);
+ ApplicationClientProtocol client = (ApplicationClientProtocol) rpc
+ .getProxy(ApplicationClientProtocol.class, rmAddress, conf);
// Get node labels collection
- GetClusterNodeLabelsResponse response =
- client.getClusterNodeLabels(GetClusterNodeLabelsRequest.newInstance());
+ GetClusterNodeLabelsResponse response = client
+ .getClusterNodeLabels(GetClusterNodeLabelsRequest.newInstance());
Assert.assertTrue(response.getNodeLabels().containsAll(
- Arrays.asList(NodeLabel.newInstance("x"), NodeLabel.newInstance("y"))));
+ Arrays.asList(labelX, labelY)));
// Get node labels mapping
- GetNodesToLabelsResponse response1 =
- client.getNodeToLabels(GetNodesToLabelsRequest.newInstance());
- Map<NodeId, Set<String>> nodeToLabels = response1.getNodeToLabels();
+ GetNodesToLabelsResponse response1 = client
+ .getNodeToLabels(GetNodesToLabelsRequest.newInstance());
+ Map<NodeId, Set<NodeLabel>> nodeToLabels = response1.getNodeToLabels();
Assert.assertTrue(nodeToLabels.keySet().containsAll(
Arrays.asList(node1, node2)));
- Assert.assertTrue(nodeToLabels.get(node1).containsAll(Arrays.asList("x")));
- Assert.assertTrue(nodeToLabels.get(node2).containsAll(Arrays.asList("y")));
-
+ Assert.assertTrue(nodeToLabels.get(node1)
+ .containsAll(Arrays.asList(labelX)));
+ Assert.assertTrue(nodeToLabels.get(node2)
+ .containsAll(Arrays.asList(labelY)));
+ // Verify whether labelX's exclusivity is false
+ for (NodeLabel x : nodeToLabels.get(node1)) {
+ Assert.assertFalse(x.isExclusive());
+ }
+ // Verify whether labelY's exclusivity is true
+ for (NodeLabel y : nodeToLabels.get(node2)) {
+ Assert.assertTrue(y.isExclusive());
+ }
+ // Below label "x" is not present in the response as exclusivity is true
+ Assert.assertFalse(nodeToLabels.get(node1).containsAll(
+ Arrays.asList(NodeLabel.newInstance("x"))));
+
rpc.stopProxy(client, conf);
rm.close();
}
@@ -1466,8 +1480,12 @@ public class TestClientRMService {
};
};
rm.start();
+
+ NodeLabel labelX = NodeLabel.newInstance("x", false);
+ NodeLabel labelY = NodeLabel.newInstance("y", false);
+ NodeLabel labelZ = NodeLabel.newInstance("z", false);
RMNodeLabelsManager labelsMgr = rm.getRMContext().getNodeLabelManager();
- labelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", "z"));
+ labelsMgr.addToCluserNodeLabels(ImmutableSet.of(labelX, labelY, labelZ));
NodeId node1A = NodeId.newInstance("host1", 1234);
NodeId node1B = NodeId.newInstance("host1", 5678);
@@ -1487,43 +1505,49 @@ public class TestClientRMService {
YarnRPC rpc = YarnRPC.create(conf);
InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
LOG.info("Connecting to ResourceManager at " + rmAddress);
- ApplicationClientProtocol client =
- (ApplicationClientProtocol) rpc.getProxy(
- ApplicationClientProtocol.class, rmAddress, conf);
+ ApplicationClientProtocol client = (ApplicationClientProtocol) rpc
+ .getProxy(ApplicationClientProtocol.class, rmAddress, conf);
// Get node labels collection
- GetClusterNodeLabelsResponse response =
- client.getClusterNodeLabels(GetClusterNodeLabelsRequest.newInstance());
+ GetClusterNodeLabelsResponse response = client
+ .getClusterNodeLabels(GetClusterNodeLabelsRequest.newInstance());
Assert.assertTrue(response.getNodeLabels().containsAll(
- Arrays.asList(NodeLabel.newInstance("x"), NodeLabel.newInstance("y"),
- NodeLabel.newInstance("z"))));
+ Arrays.asList(labelX, labelY, labelZ)));
// Get labels to nodes mapping
- GetLabelsToNodesResponse response1 =
- client.getLabelsToNodes(GetLabelsToNodesRequest.newInstance());
- Map<String, Set<NodeId>> labelsToNodes = response1.getLabelsToNodes();
- Assert.assertTrue(
- labelsToNodes.keySet().containsAll(Arrays.asList("x", "y", "z")));
- Assert.assertTrue(
- labelsToNodes.get("x").containsAll(Arrays.asList(node1A)));
- Assert.assertTrue(
- labelsToNodes.get("y").containsAll(Arrays.asList(node2A, node3A)));
- Assert.assertTrue(
- labelsToNodes.get("z").containsAll(Arrays.asList(node1B, node3B)));
+ GetLabelsToNodesResponse response1 = client
+ .getLabelsToNodes(GetLabelsToNodesRequest.newInstance());
+ Map<NodeLabel, Set<NodeId>> labelsToNodes = response1.getLabelsToNodes();
+ // Verify whether all NodeLabel's exclusivity are false
+ for (Map.Entry<NodeLabel, Set<NodeId>> nltn : labelsToNodes.entrySet()) {
+ Assert.assertFalse(nltn.getKey().isExclusive());
+ }
+ Assert.assertTrue(labelsToNodes.keySet().containsAll(
+ Arrays.asList(labelX, labelY, labelZ)));
+ Assert.assertTrue(labelsToNodes.get(labelX).containsAll(
+ Arrays.asList(node1A)));
+ Assert.assertTrue(labelsToNodes.get(labelY).containsAll(
+ Arrays.asList(node2A, node3A)));
+ Assert.assertTrue(labelsToNodes.get(labelZ).containsAll(
+ Arrays.asList(node1B, node3B)));
// Get labels to nodes mapping for specific labels
- Set<String> setlabels =
- new HashSet<String>(Arrays.asList(new String[]{"x", "z"}));
- GetLabelsToNodesResponse response2 =
- client.getLabelsToNodes(GetLabelsToNodesRequest.newInstance(setlabels));
+ Set<String> setlabels = new HashSet<String>(Arrays.asList(new String[]{"x",
+ "z"}));
+ GetLabelsToNodesResponse response2 = client
+ .getLabelsToNodes(GetLabelsToNodesRequest.newInstance(setlabels));
labelsToNodes = response2.getLabelsToNodes();
- Assert.assertTrue(
- labelsToNodes.keySet().containsAll(Arrays.asList("x", "z")));
- Assert.assertTrue(
- labelsToNodes.get("x").containsAll(Arrays.asList(node1A)));
- Assert.assertTrue(
- labelsToNodes.get("z").containsAll(Arrays.asList(node1B, node3B)));
- Assert.assertEquals(labelsToNodes.get("y"), null);
+ // Verify whether all NodeLabel's exclusivity are false
+ for (Map.Entry<NodeLabel, Set<NodeId>> nltn : labelsToNodes.entrySet()) {
+ Assert.assertFalse(nltn.getKey().isExclusive());
+ }
+ Assert.assertTrue(labelsToNodes.keySet().containsAll(
+ Arrays.asList(labelX, labelZ)));
+ Assert.assertTrue(labelsToNodes.get(labelX).containsAll(
+ Arrays.asList(node1A)));
+ Assert.assertTrue(labelsToNodes.get(labelZ).containsAll(
+ Arrays.asList(node1B, node3B)));
+ Assert.assertEquals(labelsToNodes.get(labelY), null);
rpc.stopProxy(client, conf);
rm.close();
[14/50] [abbrv] hadoop git commit: HDFS-4185. Add a metric for number
of active leases (Rakesh R via raviprak)
Posted by zj...@apache.org.
HDFS-4185. Add a metric for number of active leases (Rakesh R via raviprak)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ef6b585
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ef6b585
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ef6b585
Branch: refs/heads/YARN-2928
Commit: 7ef6b585f440ae3000d22ac479c55364136cf9f2
Parents: aacdfcb
Author: Ravi Prakash <ra...@altiscale.com>
Authored: Mon May 18 12:37:21 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:47:03 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../hdfs/server/namenode/FSNamesystem.java | 17 ++++++
.../hdfs/server/namenode/LeaseManager.java | 9 +++
.../namenode/metrics/TestNameNodeMetrics.java | 59 ++++++++++++++++++--
4 files changed, 83 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ef6b585/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7fd3495..35c3b5a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -570,6 +570,8 @@ Release 2.8.0 - UNRELEASED
HDFS-8345. Storage policy APIs must be exposed via the FileSystem
interface. (Arpit Agarwal)
+ HDFS-4185. Add a metric for number of active leases (Rakesh R via raviprak)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ef6b585/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 0fec5ee..7e5b981 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -5347,6 +5347,23 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
/**
+ * Get the number of files under construction in the system.
+ */
+ @Metric({ "NumFilesUnderConstruction",
+ "Number of files under construction" })
+ public long getNumFilesUnderConstruction() {
+ return leaseManager.countPath();
+ }
+
+ /**
+ * Get the total number of active clients holding lease in the system.
+ */
+ @Metric({ "NumActiveClients", "Number of active clients holding lease" })
+ public long getNumActiveClients() {
+ return leaseManager.countLease();
+ }
+
+ /**
* Get the total number of COMPLETE blocks in the system.
* For safe mode only complete blocks are counted.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ef6b585/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index ade2312..0806f82 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -130,6 +130,15 @@ public class LeaseManager {
@VisibleForTesting
public synchronized int countLease() {return sortedLeases.size();}
+ /** @return the number of paths contained in all leases */
+ synchronized int countPath() {
+ int count = 0;
+ for (Lease lease : sortedLeases) {
+ count += lease.getFiles().size();
+ }
+ return count;
+ }
+
/**
* Adds (or re-adds) the lease for the specified file.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ef6b585/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
index b390391..3120f85 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hdfs.server.namenode.metrics;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AUDIT_LOGGERS_KEY;
import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
import static org.apache.hadoop.test.MetricsAsserts.assertGauge;
import static org.apache.hadoop.test.MetricsAsserts.assertQuantileGauges;
@@ -29,12 +28,12 @@ import java.io.DataInputStream;
import java.io.IOException;
import java.util.Random;
import com.google.common.collect.ImmutableList;
-import com.google.common.io.Files;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.Path;
@@ -55,12 +54,10 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
-import org.apache.hadoop.hdfs.server.namenode.top.TopAuditLogger;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.MetricsSource;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.test.MetricsAsserts;
-import org.apache.hadoop.util.Time;
import org.apache.log4j.Level;
import org.junit.After;
import org.junit.Before;
@@ -567,4 +564,58 @@ public class TestNameNodeMetrics {
assertTrue(MetricsAsserts.getLongCounter("TransactionsNumOps", rbNew) >
startWriteCounter);
}
+
+ /**
+ * Test metrics indicating the number of active clients and the files under
+ * construction
+ */
+ @Test(timeout = 60000)
+ public void testNumActiveClientsAndFilesUnderConstructionMetrics()
+ throws Exception {
+ final Path file1 = getTestPath("testFileAdd1");
+ createFile(file1, 100, (short) 3);
+ assertGauge("NumActiveClients", 0L, getMetrics(NS_METRICS));
+ assertGauge("NumFilesUnderConstruction", 0L, getMetrics(NS_METRICS));
+
+ Path file2 = new Path("/testFileAdd2");
+ FSDataOutputStream output2 = fs.create(file2);
+ output2.writeBytes("Some test data");
+ assertGauge("NumActiveClients", 1L, getMetrics(NS_METRICS));
+ assertGauge("NumFilesUnderConstruction", 1L, getMetrics(NS_METRICS));
+
+ Path file3 = new Path("/testFileAdd3");
+ FSDataOutputStream output3 = fs.create(file3);
+ output3.writeBytes("Some test data");
+ assertGauge("NumActiveClients", 1L, getMetrics(NS_METRICS));
+ assertGauge("NumFilesUnderConstruction", 2L, getMetrics(NS_METRICS));
+
+ // create another DistributedFileSystem client
+ DistributedFileSystem fs1 = (DistributedFileSystem) cluster
+ .getNewFileSystemInstance(0);
+ try {
+ Path file4 = new Path("/testFileAdd4");
+ FSDataOutputStream output4 = fs1.create(file4);
+ output4.writeBytes("Some test data");
+ assertGauge("NumActiveClients", 2L, getMetrics(NS_METRICS));
+ assertGauge("NumFilesUnderConstruction", 3L, getMetrics(NS_METRICS));
+
+ Path file5 = new Path("/testFileAdd35");
+ FSDataOutputStream output5 = fs1.create(file5);
+ output5.writeBytes("Some test data");
+ assertGauge("NumActiveClients", 2L, getMetrics(NS_METRICS));
+ assertGauge("NumFilesUnderConstruction", 4L, getMetrics(NS_METRICS));
+
+ output2.close();
+ output3.close();
+ assertGauge("NumActiveClients", 1L, getMetrics(NS_METRICS));
+ assertGauge("NumFilesUnderConstruction", 2L, getMetrics(NS_METRICS));
+
+ output4.close();
+ output5.close();
+ assertGauge("NumActiveClients", 0L, getMetrics(NS_METRICS));
+ assertGauge("NumFilesUnderConstruction", 0L, getMetrics(NS_METRICS));
+ } finally {
+ fs1.close();
+ }
+ }
}
[11/50] [abbrv] hadoop git commit: HADOOP-11884. test-patch.sh should
pull the real findbugs version (Kengo Seki via aw)
Posted by zj...@apache.org.
HADOOP-11884. test-patch.sh should pull the real findbugs version (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bb44e04a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bb44e04a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bb44e04a
Branch: refs/heads/YARN-2928
Commit: bb44e04a0d49695b6c2cca38d6ee8238c5ebd4c7
Parents: 01006c9
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon May 18 16:08:49 2015 +0000
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:47:02 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.sh | 5 +++--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb44e04a/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 9cc5bb0..00a638c 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -1859,8 +1859,6 @@ function check_findbugs
return 1
fi
- findbugs_version=$("${FINDBUGS_HOME}/bin/findbugs" -version)
-
for module in ${modules}
do
pushd "${module}" >/dev/null
@@ -1872,6 +1870,9 @@ function check_findbugs
popd >/dev/null
done
+ #shellcheck disable=SC2016
+ findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt")
+
if [[ ${rc} -ne 0 ]]; then
add_jira_table -1 findbugs "The patch appears to cause Findbugs (version ${findbugs_version}) to fail."
return 1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb44e04a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2138334..1c2cdaa 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -575,6 +575,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11939. Deprecate DistCpV1 and Logalyzer.
(Brahma Reddy Battula via aajisaka)
+ HADOOP-11884. test-patch.sh should pull the real findbugs version
+ (Kengo Seki via aw)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
[24/50] [abbrv] hadoop git commit: HADOOP-10971. Add -C flag to make
`hadoop fs -ls` print filenames only. Contributed by Kengo Seki.
Posted by zj...@apache.org.
HADOOP-10971. Add -C flag to make `hadoop fs -ls` print filenames only. Contributed by Kengo Seki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d48e623f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d48e623f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d48e623f
Branch: refs/heads/YARN-2928
Commit: d48e623fee56c2e4488e0095719dcd189807b3ac
Parents: 8c1572e
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue May 19 17:25:27 2015 +0900
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:52:08 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../java/org/apache/hadoop/fs/shell/Ls.java | 36 +++++++++--
.../src/site/markdown/FileSystemShell.md | 3 +-
.../java/org/apache/hadoop/fs/shell/TestLs.java | 67 ++++++++++++++++++++
.../src/test/resources/testConf.xml | 6 +-
5 files changed, 106 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d48e623f/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index cf09c5f..8ce77b6 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -488,6 +488,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11949. Add user-provided plugins to test-patch (Sean Busbey via aw)
+ HADOOP-10971. Add -C flag to make `hadoop fs -ls` print filenames only.
+ (Kengo Seki via aajisaka)
+
IMPROVEMENTS
HADOOP-6842. "hadoop fs -text" does not give a useful text representation
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d48e623f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
index 8ef6d5e..d5c52ed 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
@@ -43,6 +43,7 @@ class Ls extends FsCommand {
factory.addClass(Lsr.class, "-lsr");
}
+ private static final String OPTION_PATHONLY = "C";
private static final String OPTION_DIRECTORY = "d";
private static final String OPTION_HUMAN = "h";
private static final String OPTION_RECURSIVE = "R";
@@ -52,10 +53,10 @@ class Ls extends FsCommand {
private static final String OPTION_SIZE = "S";
public static final String NAME = "ls";
- public static final String USAGE = "[-" + OPTION_DIRECTORY + "] [-"
- + OPTION_HUMAN + "] " + "[-" + OPTION_RECURSIVE + "] [-" + OPTION_MTIME
- + "] [-" + OPTION_SIZE + "] [-" + OPTION_REVERSE + "] " + "[-"
- + OPTION_ATIME + "] [<path> ...]";
+ public static final String USAGE = "[-" + OPTION_PATHONLY + "] [-"
+ + OPTION_DIRECTORY + "] [-" + OPTION_HUMAN + "] [-" + OPTION_RECURSIVE
+ + "] [-" + OPTION_MTIME + "] [-" + OPTION_SIZE + "] [-" + OPTION_REVERSE
+ + "] [-" + OPTION_ATIME + "] [<path> ...]";
public static final String DESCRIPTION =
"List the contents that match the specified file pattern. If " +
@@ -67,6 +68,8 @@ class Ls extends FsCommand {
"\tpermissions - userId groupId sizeOfDirectory(in bytes) modificationDate(yyyy-MM-dd HH:mm) directoryName\n\n" +
"and file entries are of the form:\n" +
"\tpermissions numberOfReplicas userId groupId sizeOfFile(in bytes) modificationDate(yyyy-MM-dd HH:mm) fileName\n\n" +
+ " -" + OPTION_PATHONLY +
+ " Display the paths of files and directories only.\n" +
" -" + OPTION_DIRECTORY +
" Directories are listed as plain files.\n" +
" -" + OPTION_HUMAN +
@@ -89,6 +92,7 @@ class Ls extends FsCommand {
protected int maxRepl = 3, maxLen = 10, maxOwner = 0, maxGroup = 0;
protected String lineFormat;
+ private boolean pathOnly;
protected boolean dirRecurse;
private boolean orderReverse;
private boolean orderTime;
@@ -107,10 +111,11 @@ class Ls extends FsCommand {
@Override
protected void processOptions(LinkedList<String> args)
throws IOException {
- CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE,
+ CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, OPTION_PATHONLY,
OPTION_DIRECTORY, OPTION_HUMAN, OPTION_RECURSIVE, OPTION_REVERSE,
OPTION_MTIME, OPTION_SIZE, OPTION_ATIME);
cf.parse(args);
+ pathOnly = cf.getOpt(OPTION_PATHONLY);
dirRecurse = !cf.getOpt(OPTION_DIRECTORY);
setRecursive(cf.getOpt(OPTION_RECURSIVE) && dirRecurse);
humanReadable = cf.getOpt(OPTION_HUMAN);
@@ -124,6 +129,15 @@ class Ls extends FsCommand {
}
/**
+ * Should display only paths of files and directories.
+ * @return true display paths only, false display all fields
+ */
+ @InterfaceAudience.Private
+ boolean isPathOnly() {
+ return this.pathOnly;
+ }
+
+ /**
* Should the contents of the directory be shown or just the directory?
* @return true if directory contents, false if just directory
*/
@@ -191,15 +205,23 @@ class Ls extends FsCommand {
protected void processPaths(PathData parent, PathData ... items)
throws IOException {
if (parent != null && !isRecursive() && items.length != 0) {
- out.println("Found " + items.length + " items");
+ if (!pathOnly) {
+ out.println("Found " + items.length + " items");
+ }
Arrays.sort(items, getOrderComparator());
}
- adjustColumnWidths(items);
+ if (!pathOnly) {
+ adjustColumnWidths(items);
+ }
super.processPaths(parent, items);
}
@Override
protected void processPath(PathData item) throws IOException {
+ if (pathOnly) {
+ out.println(item.toString());
+ return;
+ }
FileStatus stat = item.stat;
String line = String.format(lineFormat,
(stat.isDirectory() ? "d" : "-"),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d48e623f/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 2920e01..8ecc24a 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -382,10 +382,11 @@ Return usage output.
ls
----
-Usage: `hadoop fs -ls [-d] [-h] [-R] [-t] [-S] [-r] [-u] <args> `
+Usage: `hadoop fs -ls [-C] [-d] [-h] [-R] [-t] [-S] [-r] [-u] <args> `
Options:
+* -C: Display the paths of files and directories only.
* -d: Directories are listed as plain files.
* -h: Format file sizes in a human-readable fashion (eg 64.0m instead of 67108864).
* -R: Recursively list subdirectories encountered.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d48e623f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
index 66403db..4a9103f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
@@ -74,6 +74,24 @@ public class TestLs {
LinkedList<String> options = new LinkedList<String>();
Ls ls = new Ls();
ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
+ assertTrue(ls.isDirRecurse());
+ assertFalse(ls.isHumanReadable());
+ assertFalse(ls.isRecursive());
+ assertFalse(ls.isOrderReverse());
+ assertFalse(ls.isOrderSize());
+ assertFalse(ls.isOrderTime());
+ assertFalse(ls.isUseAtime());
+ }
+
+ // check the -C option is recognised
+ @Test
+ public void processOptionsPathOnly() throws IOException {
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-C");
+ Ls ls = new Ls();
+ ls.processOptions(options);
+ assertTrue(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@@ -90,6 +108,7 @@ public class TestLs {
options.add("-d");
Ls ls = new Ls();
ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
assertFalse(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@@ -106,6 +125,7 @@ public class TestLs {
options.add("-h");
Ls ls = new Ls();
ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertTrue(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@@ -122,6 +142,7 @@ public class TestLs {
options.add("-R");
Ls ls = new Ls();
ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertTrue(ls.isRecursive());
@@ -138,6 +159,7 @@ public class TestLs {
options.add("-r");
Ls ls = new Ls();
ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@@ -154,6 +176,7 @@ public class TestLs {
options.add("-S");
Ls ls = new Ls();
ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@@ -170,6 +193,7 @@ public class TestLs {
options.add("-t");
Ls ls = new Ls();
ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@@ -187,6 +211,7 @@ public class TestLs {
options.add("-S");
Ls ls = new Ls();
ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@@ -205,6 +230,7 @@ public class TestLs {
options.add("-r");
Ls ls = new Ls();
ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@@ -221,6 +247,7 @@ public class TestLs {
options.add("-u");
Ls ls = new Ls();
ls.processOptions(options);
+ assertFalse(ls.isPathOnly());
assertTrue(ls.isDirRecurse());
assertFalse(ls.isHumanReadable());
assertFalse(ls.isRecursive());
@@ -234,6 +261,7 @@ public class TestLs {
@Test
public void processOptionsAll() throws IOException {
LinkedList<String> options = new LinkedList<String>();
+ options.add("-C"); // show file path only
options.add("-d"); // directory
options.add("-h"); // human readable
options.add("-R"); // recursive
@@ -243,6 +271,7 @@ public class TestLs {
options.add("-u"); // show atime
Ls ls = new Ls();
ls.processOptions(options);
+ assertTrue(ls.isPathOnly());
assertFalse(ls.isDirRecurse());
assertTrue(ls.isHumanReadable());
assertFalse(ls.isRecursive()); // -d overrules -R
@@ -981,6 +1010,44 @@ public class TestLs {
verifyNoMoreInteractions(out);
}
+ // check path only display (-C option)
+ @Test
+ public void processPathDirectoryPathOnly() throws IOException {
+ TestFile testfile01 = new TestFile("testDirectory", "testFile01");
+ TestFile testfile02 = new TestFile("testDirectory", "testFile02");
+ TestFile testfile03 = new TestFile("testDirectory", "testFile03");
+ TestFile testfile04 = new TestFile("testDirectory", "testFile04");
+ TestFile testfile05 = new TestFile("testDirectory", "testFile05");
+ TestFile testfile06 = new TestFile("testDirectory", "testFile06");
+
+ TestFile testDir = new TestFile("", "testDirectory");
+ testDir.setIsDir(true);
+ testDir.addContents(testfile01, testfile02, testfile03, testfile04,
+ testfile05, testfile06);
+
+ LinkedList<PathData> pathData = new LinkedList<PathData>();
+ pathData.add(testDir.getPathData());
+
+ PrintStream out = mock(PrintStream.class);
+
+ Ls ls = new Ls();
+ ls.out = out;
+
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-C");
+ ls.processOptions(options);
+
+ ls.processArguments(pathData);
+ InOrder inOrder = inOrder(out);
+ inOrder.verify(out).println(testfile01.getPath().toString());
+ inOrder.verify(out).println(testfile02.getPath().toString());
+ inOrder.verify(out).println(testfile03.getPath().toString());
+ inOrder.verify(out).println(testfile04.getPath().toString());
+ inOrder.verify(out).println(testfile05.getPath().toString());
+ inOrder.verify(out).println(testfile06.getPath().toString());
+ verifyNoMoreInteractions(out);
+ }
+
// check the deprecated flag isn't set
@Test
public void isDeprecated() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d48e623f/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index 57cce14..62afff2 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -54,7 +54,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-ls \[-d\] \[-h\] \[-R\] \[-t\] \[-S\] \[-r\] \[-u\] \[<path> \.\.\.\] :( |\t)*</expected-output>
+ <expected-output>^-ls \[-C\] \[-d\] \[-h\] \[-R\] \[-t\] \[-S\] \[-r\] \[-u\] \[<path> \.\.\.\] :( |\t)*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@@ -94,6 +94,10 @@
</comparator>
<comparator>
<type>RegexpComparator</type>
+ <expected-output>^\s*-C\s+Display the paths of files and directories only\.( )*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
<expected-output>^\s*-d\s+Directories are listed as plain files\.( )*</expected-output>
</comparator>
<comparator>
[45/50] [abbrv] hadoop git commit: YARN-3681. yarn cmd says "could
not find main class 'queue'" in windows. Contributed by Craig Welch and Varun
Saxena
Posted by zj...@apache.org.
YARN-3681. yarn cmd says "could not find main class 'queue'" in windows.
Contributed by Craig Welch and Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f416078c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f416078c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f416078c
Branch: refs/heads/YARN-2928
Commit: f416078c05ebe2ba629995a16285257f29162938
Parents: f638f3c
Author: Xuan <xg...@apache.org>
Authored: Wed May 20 14:21:12 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 12:42:57 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f416078c/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 93e78af..f1b5a2d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -607,6 +607,9 @@ Release 2.7.1 - UNRELEASED
YARN-2918. RM should not fail on startup if queue's configured labels do
not exist in cluster-node-labels. (Wangda Tan via jianhe)
+ YARN-3681. yarn cmd says "could not find main class 'queue'" in windows.
+ (Craig Welch and Varun Saxena via xgong)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f416078c/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
index c29ee53..91c90fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
@@ -150,7 +150,7 @@ if "%1" == "--loglevel" (
)
set yarncommands=resourcemanager nodemanager proxyserver rmadmin version jar ^
- application applicationattempt container node logs daemonlog historyserver ^
+ application applicationattempt container node queue logs daemonlog historyserver ^
timelineserver classpath
for %%i in ( %yarncommands% ) do (
if %yarn-command% == %%i set yarncommand=true
[17/50] [abbrv] hadoop git commit: HADOOP-11949. Add user-provided
plugins to test-patch (Sean Busbey via aw)
Posted by zj...@apache.org.
HADOOP-11949. Add user-provided plugins to test-patch (Sean Busbey via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2197cd57
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2197cd57
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2197cd57
Branch: refs/heads/YARN-2928
Commit: 2197cd5787e3d4dec3b2b743b6749aa8579ea910
Parents: f120e13
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon May 18 17:06:31 2015 +0000
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 11:47:03 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.sh | 27 +++++++++++++++++---
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
2 files changed, 25 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2197cd57/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index ae74c5b..57fd657 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -40,6 +40,9 @@ function setup_defaults
BASEDIR=$(pwd)
RELOCATE_PATCH_DIR=false
+ USER_PLUGIN_DIR=""
+ LOAD_SYSTEM_PLUGINS=true
+
FINDBUGS_HOME=${FINDBUGS_HOME:-}
ECLIPSE_HOME=${ECLIPSE_HOME:-}
BUILD_NATIVE=${BUILD_NATIVE:-true}
@@ -586,9 +589,11 @@ function hadoop_usage
echo "--modulelist=<list> Specify additional modules to test (comma delimited)"
echo "--offline Avoid connecting to the Internet"
echo "--patch-dir=<dir> The directory for working and output files (default '/tmp/${PROJECT_NAME}-test-patch/pid')"
+ echo "--plugins=<dir> A directory of user provided plugins. see test-patch.d for examples (default empty)"
echo "--project=<name> The short name for project currently using test-patch (default 'hadoop')"
echo "--resetrepo Forcibly clean the repo"
echo "--run-tests Run all relevant tests below the base directory"
+ echo "--skip-system-plugins Do not load plugins from ${BINDIR}/test-patch.d"
echo "--testlist=<list> Specify which subsystem tests to use (comma delimited)"
echo "Shell binary overrides:"
@@ -706,6 +711,9 @@ function parse_args
--patch-dir=*)
USER_PATCH_DIR=${i#*=}
;;
+ --plugins=*)
+ USER_PLUGIN_DIR=${i#*=}
+ ;;
--project=*)
PROJECT_NAME=${i#*=}
;;
@@ -723,6 +731,9 @@ function parse_args
--run-tests)
RUN_TESTS=true
;;
+ --skip-system-plugins)
+ LOAD_SYSTEM_PLUGINS=false
+ ;;
--testlist=*)
testlist=${i#*=}
testlist=${testlist//,/ }
@@ -2523,17 +2534,25 @@ function runtests
done
}
-## @description Import content from test-patch.d
+## @description Import content from test-patch.d and optionally
+## @description from user provided plugin directory
## @audience private
## @stability evolving
## @replaceable no
function importplugins
{
local i
- local files
+ local files=()
+
+ if [[ ${LOAD_SYSTEM_PLUGINS} == "true" ]]; then
+ if [[ -d "${BINDIR}/test-patch.d" ]]; then
+ files=(${BINDIR}/test-patch.d/*.sh)
+ fi
+ fi
- if [[ -d "${BINDIR}/test-patch.d" ]]; then
- files=(${BINDIR}/test-patch.d/*.sh)
+ if [[ -n "${USER_PLUGIN_DIR}" && -d "${USER_PLUGIN_DIR}" ]]; then
+ hadoop_debug "Loading user provided plugins from ${USER_PLUGIN_DIR}"
+ files=("${files[@]}" ${USER_PLUGIN_DIR}/*.sh)
fi
for i in "${files[@]}"; do
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2197cd57/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 8f66072..324434b 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -486,6 +486,8 @@ Release 2.8.0 - UNRELEASED
HADOOP-11843. Make setting up the build environment easier.
(Niels Basjes via cnauroth)
+ HADOOP-11949. Add user-provided plugins to test-patch (Sean Busbey via aw)
+
IMPROVEMENTS
HADOOP-6842. "hadoop fs -text" does not give a useful text representation