You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2017/11/24 10:43:19 UTC
[08/49] hadoop git commit: HDFS-12730. Verify open files captured in
the snapshots across config disable and enable.
HDFS-12730. Verify open files captured in the snapshots across config disable and enable.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9fb4effd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9fb4effd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9fb4effd
Branch: refs/heads/YARN-5881
Commit: 9fb4effd2c4de2d83b667a43e8798315e85ff79b
Parents: 6f9d7a1
Author: Manoj Govindassamy <ma...@apache.org>
Authored: Mon Nov 20 10:28:12 2017 -0800
Committer: Manoj Govindassamy <ma...@apache.org>
Committed: Mon Nov 20 10:28:12 2017 -0800
----------------------------------------------------------------------
.../namenode/snapshot/SnapshotManager.java | 8 +-
.../snapshot/TestOpenFilesWithSnapshot.java | 117 +++++++++++++++++++
2 files changed, 124 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fb4effd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
index 89033ef..58a218e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
@@ -35,6 +35,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import javax.management.ObjectName;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -75,7 +76,7 @@ public class SnapshotManager implements SnapshotStatsMXBean {
public static final Log LOG = LogFactory.getLog(SnapshotManager.class);
private final FSDirectory fsdir;
- private final boolean captureOpenFiles;
+ private boolean captureOpenFiles;
/**
* If skipCaptureAccessTimeOnlyChange is set to true, if accessTime
* of a file changed but there is no other modification made to the file,
@@ -121,6 +122,11 @@ public class SnapshotManager implements SnapshotStatsMXBean {
+ snapshotDiffAllowSnapRootDescendant);
}
+ @VisibleForTesting
+ void setCaptureOpenFiles(boolean captureOpenFiles) {
+ this.captureOpenFiles = captureOpenFiles;
+ }
+
/**
* @return skipCaptureAccessTimeOnlyChange
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fb4effd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
index 537612c..be118a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
@@ -843,6 +843,123 @@ public class TestOpenFilesWithSnapshot {
hbaseFileCksumBeforeTruncate, hbaseFileCksumS3);
}
+ private Path createSnapshot(Path snapRootDir, String snapName,
+ String fileName) throws Exception {
+ final Path snap1Dir = SnapshotTestHelper.createSnapshot(
+ fs, snapRootDir, snapName);
+ return new Path(snap1Dir, fileName);
+ }
+
+ private void verifyFileSize(long fileSize, Path... filePaths) throws
+ IOException {
+ for (Path filePath : filePaths) {
+ Assert.assertEquals(fileSize, fs.getFileStatus(filePath).getLen());
+ }
+ }
+
+ /**
+ * Verify open files captured in the snapshots across config disable
+ * and enable.
+ */
+ @Test
+ public void testOpenFilesWithMixedConfig() throws Exception {
+ final Path snapRootDir = new Path("/level_0_A");
+ final String flumeFileName = "flume.log";
+ final String snap1Name = "s1";
+ final String snap2Name = "s2";
+ final String snap3Name = "s3";
+ final String snap4Name = "s4";
+ final String snap5Name = "s5";
+
+ // Create files and open streams
+ final Path flumeFile = new Path(snapRootDir, flumeFileName);
+ createFile(flumeFile);
+ FSDataOutputStream flumeOutputStream = fs.append(flumeFile);
+
+ // 1. Disable capture open files
+ cluster.getNameNode().getNamesystem()
+ .getSnapshotManager().setCaptureOpenFiles(false);
+
+ // Create Snapshot S1
+ final Path flumeS1Path = createSnapshot(snapRootDir,
+ snap1Name, flumeFileName);
+
+ // Verify if Snap S1 file length is same as the the current versions
+ verifyFileSize(FILELEN, flumeS1Path);
+
+ // Write more data to files
+ long flumeFileWrittenDataLength = FILELEN;
+ int newWriteLength = (int) (BLOCKSIZE * 1.5);
+ byte[] buf = new byte[newWriteLength];
+ Random random = new Random();
+ random.nextBytes(buf);
+ flumeFileWrittenDataLength += writeToStream(flumeOutputStream, buf);
+
+ // Create Snapshot S2
+ final Path flumeS2Path = createSnapshot(snapRootDir,
+ snap2Name, flumeFileName);
+
+ // Since capture open files was disabled, all snapshots paths
+ // and the current version should have same file lengths.
+ verifyFileSize(flumeFileWrittenDataLength,
+ flumeFile, flumeS2Path, flumeS1Path);
+
+ // 2. Enable capture open files
+ cluster.getNameNode().getNamesystem()
+ .getSnapshotManager() .setCaptureOpenFiles(true);
+
+ // Write more data to files
+ flumeFileWrittenDataLength += writeToStream(flumeOutputStream, buf);
+ long flumeFileLengthAfterS3 = flumeFileWrittenDataLength;
+
+ // Create Snapshot S3
+ final Path flumeS3Path = createSnapshot(snapRootDir,
+ snap3Name, flumeFileName);
+
+ // Since open files captured in the previous snapshots were with config
+ // disabled, their file lengths are now same as the current version.
+ // With the config turned on, any new data written to the open files
+ // will no more reflect in the current version or old snapshot paths.
+ verifyFileSize(flumeFileWrittenDataLength, flumeFile, flumeS3Path,
+ flumeS2Path, flumeS1Path);
+
+ // Write more data to files
+ flumeFileWrittenDataLength += writeToStream(flumeOutputStream, buf);
+
+ // Create Snapshot S4
+ final Path flumeS4Path = createSnapshot(snapRootDir,
+ snap4Name, flumeFileName);
+
+ // Verify S4 has the latest data
+ verifyFileSize(flumeFileWrittenDataLength, flumeFile, flumeS4Path);
+
+ // But, open files captured as of Snapshot S3 and before should
+ // have their old file lengths intact.
+ verifyFileSize(flumeFileLengthAfterS3, flumeS3Path,
+ flumeS2Path, flumeS1Path);
+
+ long flumeFileLengthAfterS4 = flumeFileWrittenDataLength;
+
+ // 3. Disable capture open files
+ cluster.getNameNode().getNamesystem()
+ .getSnapshotManager() .setCaptureOpenFiles(false);
+
+ // Create Snapshot S5
+ final Path flumeS5Path = createSnapshot(snapRootDir,
+ snap5Name, flumeFileName);
+
+ flumeFileWrittenDataLength += writeToStream(flumeOutputStream, buf);
+
+ // Since capture open files was disabled, any snapshots taken after the
+ // config change and the current version should have same file lengths
+ // for the open files.
+ verifyFileSize(flumeFileWrittenDataLength, flumeFile, flumeS5Path);
+
+ // But, the old snapshots taken before the config disable should
+ // continue to be consistent.
+ verifyFileSize(flumeFileLengthAfterS4, flumeS4Path);
+ }
+
private void restartNameNode() throws Exception {
cluster.triggerBlockReports();
NameNode nameNode = cluster.getNameNode();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org