You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ma...@apache.org on 2017/08/29 18:28:04 UTC
hadoop git commit: HDFS-11912. Snapshot functionality test along with
randomized file IO operations. (George Huang via Manoj Govindassamy)
Repository: hadoop
Updated Branches:
refs/heads/trunk 9374f3882 -> 63fc1b0b6
HDFS-11912. Snapshot functionality test along with randomized file IO operations. (George Huang via Manoj Govindassamy)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/63fc1b0b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/63fc1b0b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/63fc1b0b
Branch: refs/heads/trunk
Commit: 63fc1b0b6db8fb3730dfc8c986aac09d426272e1
Parents: 9374f38
Author: Manoj Govindassamy <ma...@apache.org>
Authored: Tue Aug 29 11:27:47 2017 -0700
Committer: Manoj Govindassamy <ma...@apache.org>
Committed: Tue Aug 29 11:27:47 2017 -0700
----------------------------------------------------------------------
.../snapshot/TestRandomOpsWithSnapshots.java | 691 +++++++++++++++++++
1 file changed, 691 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63fc1b0b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRandomOpsWithSnapshots.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRandomOpsWithSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRandomOpsWithSnapshots.java
new file mode 100644
index 0000000..80af690
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRandomOpsWithSnapshots.java
@@ -0,0 +1,691 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode.snapshot;
+
+import com.google.common.base.Supplier;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.SnapshotException;
+import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.TimeoutException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Testing random FileSystem operations with random Snapshot operations.
+ */
+public class TestRandomOpsWithSnapshots {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestRandomOpsWithSnapshots.class);
+
+ private static final short REPL = 3;
+ private static final long BLOCKSIZE = 1024;
+
+ private static final int TOTAL_FILECOUNT = 250;
+ private static final int MAX_NUM_ITERATIONS = 10;
+ private static final int MAX_NUM_FILESYSTEM_OPERATIONS = 50;
+ private static final int MAX_NUM_SNAPSHOT_OPERATIONS = 50;
+ private static final int MAX_NUM_SUB_DIRECTORIES_LEVEL = 10;
+ private static final int MAX_NUM_FILE_LENGTH = 100;
+ private static final int MIN_NUM_OPERATIONS = 25;
+
+ private static final String TESTDIRSTRING = "/testDir";
+ private static final String WITNESSDIRSTRING = "/WITNESSDIR";
+ private static final Path TESTDIR = new Path(TESTDIRSTRING);
+ private static final Path WITNESSDIR = new Path(WITNESSDIRSTRING);
+ private static List<Path> snapshottableDirectories = new ArrayList<Path>();
+ private static Map<Path, ArrayList<String>> pathToSnapshotsMap =
+ new HashMap<Path, ArrayList<String>>();
+
+ private static final Configuration CONFIG = new Configuration();
+ private MiniDFSCluster cluster;
+ private DistributedFileSystem hdfs;
+ private static Random generator = null;
+
+ private int numberFileCreated = 0;
+ private int numberFileDeleted = 0;
+ private int numberFileRenamed = 0;
+
+ private int numberDirectoryCreated = 0;
+ private int numberDirectoryDeleted = 0;
+ private int numberDirectoryRenamed = 0;
+
+ private int numberSnapshotCreated = 0;
+ private int numberSnapshotDeleted = 0;
+ private int numberSnapshotRenamed = 0;
+
+ // Operation directories
+ private enum OperationDirectories {
+ TestDir,
+ WitnessDir;
+ }
+
+ // Operation type
+ private enum OperationType {
+ FileSystem,
+ Snapshot;
+ }
+
+ // FileSystem & Snapshot operation
+ private enum Operations {
+ FileSystem_CreateFile(2 /*operation weight*/, OperationType.FileSystem),
+ FileSystem_DeleteFile(2, OperationType.FileSystem),
+ FileSystem_RenameFile(2, OperationType.FileSystem),
+ FileSystem_CreateDir(1, OperationType.FileSystem),
+ FileSystem_DeleteDir(1, OperationType.FileSystem),
+ FileSystem_RenameDir(2, OperationType.FileSystem),
+
+ Snapshot_CreateSnapshot(5, OperationType.Snapshot),
+ Snapshot_DeleteSnapshot(3, OperationType.Snapshot),
+ Snapshot_RenameSnapshot(2, OperationType.Snapshot);
+
+ private int weight;
+ private OperationType operationType;
+
+ Operations(int weight, OperationType type) {
+ this.weight = weight;
+ this.operationType = type;
+ }
+
+ private int getWeight() {
+ return weight;
+ }
+
+ private static final Operations[] VALUES = values();
+
+ private static int sumWeights(OperationType type) {
+ int sum = 0;
+ for (Operations value: VALUES) {
+ if (value.operationType == type) {
+ sum += value.getWeight();
+ }
+ }
+ return sum;
+ }
+
+ private static final int TOTAL_WEIGHT_FILESYSTEM =
+ sumWeights(OperationType.FileSystem);
+
+ private static final int TOTAL_WEIGHT_SNAPSHOT =
+ sumWeights(OperationType.Snapshot);
+
+ public static Operations getRandomOperation(OperationType type) {
+ int randomNum = 0;
+ Operations randomOperation = null;
+ switch (type) {
+ case FileSystem:
+ randomNum = generator.nextInt(TOTAL_WEIGHT_FILESYSTEM);
+ break;
+ case Snapshot:
+ randomNum = generator.nextInt(TOTAL_WEIGHT_SNAPSHOT);
+ break;
+ default:
+ break;
+ }
+ int currentWeightSum = 0;
+ for (Operations currentValue: VALUES) {
+ if (currentValue.operationType == type) {
+ if (randomNum <= (currentWeightSum + currentValue.getWeight())) {
+ randomOperation = currentValue;
+ break;
+ }
+ currentWeightSum += currentValue.getWeight();
+ }
+ }
+ return randomOperation;
+ }
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ CONFIG.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCKSIZE);
+ cluster = new MiniDFSCluster.Builder(CONFIG).numDataNodes(REPL).
+ format(true).build();
+ cluster.waitActive();
+
+ hdfs = cluster.getFileSystem();
+ hdfs.mkdirs(TESTDIR);
+ hdfs.mkdirs(WITNESSDIR);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
+ /*
+ * Random file system operations with snapshot operations in between.
+ */
+ @Test(timeout = 900000)
+ public void testRandomOperationsWithSnapshots()
+ throws IOException, InterruptedException, TimeoutException {
+ // Set
+ long seed = System.currentTimeMillis();
+ LOG.info("testRandomOperationsWithSnapshots, seed to be used: " + seed);
+ generator = new Random(seed);
+
+ int fileLen = generator.nextInt(MAX_NUM_FILE_LENGTH);
+ createFiles(TESTDIRSTRING, fileLen);
+
+ // Get list of snapshottable directories
+ SnapshottableDirectoryStatus[] snapshottableDirectoryStatus =
+ hdfs.getSnapshottableDirListing();
+ for (SnapshottableDirectoryStatus ssds : snapshottableDirectoryStatus) {
+ snapshottableDirectories.add(ssds.getFullPath());
+ }
+
+ if (snapshottableDirectories.size() == 0) {
+ hdfs.allowSnapshot(hdfs.getHomeDirectory());
+ snapshottableDirectories.add(hdfs.getHomeDirectory());
+ }
+
+ int numberOfIterations = generator.nextInt(MAX_NUM_ITERATIONS);
+ LOG.info("Number of iterations: " + numberOfIterations);
+
+ int numberFileSystemOperations = generator.nextInt(
+ MAX_NUM_FILESYSTEM_OPERATIONS-MIN_NUM_OPERATIONS+1)+MIN_NUM_OPERATIONS;
+ LOG.info("Number of FileSystem operations: "+ numberFileSystemOperations);
+
+ int numberSnapshotOperations = generator.nextInt(
+ MAX_NUM_SNAPSHOT_OPERATIONS-MIN_NUM_OPERATIONS)+MIN_NUM_OPERATIONS;
+ LOG.info("Number of Snapshot operations: " + numberSnapshotOperations);
+
+ // Act && Verify
+ randomOperationsWithSnapshots(numberOfIterations,
+ numberFileSystemOperations, numberSnapshotOperations);
+ }
+
+ /*
+ * Based on input we're performing:
+ * random number of file system operations
+ * random number of snapshot operations
+ * restart name node making sure fsimage can be loaded successfully.
+ */
+ public void randomOperationsWithSnapshots(int numberOfIterations,
+ int numberFileSystemOperations,
+ int numberSnapshotOperations)
+ throws IOException, InterruptedException, TimeoutException {
+ // random number of iterations
+ for (int i = 0; i < numberOfIterations; i++) {
+ // random number of FileSystem operations
+ for (int j = 0; j < numberFileSystemOperations; j++) {
+ Operations fsOperation =
+ Operations.getRandomOperation(OperationType.FileSystem);
+ LOG.info("fsOperation: " + fsOperation);
+ switch (fsOperation) {
+ case FileSystem_CreateDir:
+ createTestDir();
+ break;
+
+ case FileSystem_DeleteDir:
+ deleteTestDir();
+ break;
+
+ case FileSystem_RenameDir:
+ renameTestDir();
+ break;
+
+ case FileSystem_CreateFile:
+ createTestFile();
+ break;
+
+ case FileSystem_DeleteFile:
+ deleteTestFile();
+ break;
+
+ case FileSystem_RenameFile:
+ renameTestFile();
+ break;
+
+ default:
+ assertNull("Invalid FileSystem operation", fsOperation);
+ break;
+ }
+ }
+
+ // random number of Snapshot operations
+ for (int k = 0; k < numberSnapshotOperations; k++) {
+ Operations snapshotOperation =
+ Operations.getRandomOperation(OperationType.Snapshot);
+ LOG.info("snapshotOperation: " + snapshotOperation);
+
+ switch (snapshotOperation) {
+ case Snapshot_CreateSnapshot:
+ createSnapshot();
+ break;
+
+ case Snapshot_DeleteSnapshot:
+ deleteSnapshot();
+ break;
+
+ case Snapshot_RenameSnapshot:
+ renameSnapshot();
+ break;
+
+ default:
+ assertNull("Invalid Snapshot operation", snapshotOperation);
+ break;
+ }
+ }
+ // Verification
+ checkClusterHealth();
+ }
+ }
+
+ /* Create a new test directory. */
+ private void createTestDir() throws IOException {
+ if (snapshottableDirectories.size() > 0) {
+ int index = generator.nextInt(snapshottableDirectories.size());
+ Path parentDir = snapshottableDirectories.get(index);
+ Path newDir = new Path(parentDir, "createTestDir_" +
+ UUID.randomUUID().toString());
+
+ for (OperationDirectories dir : OperationDirectories.values()) {
+ if (dir == OperationDirectories.WitnessDir) {
+ newDir = new Path(getNewPathString(newDir.toString(),
+ TESTDIRSTRING, WITNESSDIRSTRING));
+ }
+ hdfs.mkdirs(newDir);
+ assertTrue("Directory exists", hdfs.exists(newDir));
+ LOG.info("Directory created: " + newDir);
+ numberDirectoryCreated++;
+ }
+ }
+ }
+
+ /* Delete an existing test directory. */
+ private void deleteTestDir() throws IOException {
+ if (snapshottableDirectories.size() > 0) {
+ int index = generator.nextInt(snapshottableDirectories.size());
+ Path deleteDir = snapshottableDirectories.get(index);
+
+ if (!pathToSnapshotsMap.containsKey(deleteDir)) {
+ boolean isWitnessDir = false;
+ for (OperationDirectories dir : OperationDirectories.values()) {
+ if (dir == OperationDirectories.WitnessDir) {
+ isWitnessDir = true;
+ deleteDir = new Path(getNewPathString(deleteDir.toString(),
+ TESTDIRSTRING, WITNESSDIRSTRING));
+ }
+ hdfs.delete(deleteDir, true);
+ assertFalse("Directory does not exist", hdfs.exists(deleteDir));
+ if (!isWitnessDir) {
+ snapshottableDirectories.remove(deleteDir);
+ }
+ LOG.info("Directory removed: " + deleteDir);
+ numberDirectoryDeleted++;
+ }
+ }
+ }
+ }
+
+ /* Rename an existing test directory. */
+ private void renameTestDir() throws IOException {
+ if (snapshottableDirectories.size() > 0) {
+ int index = generator.nextInt(snapshottableDirectories.size());
+ Path oldDir = snapshottableDirectories.get(index);
+
+ if (!pathToSnapshotsMap.containsKey(oldDir)) {
+ Path newDir = oldDir.suffix("_renameDir+"+UUID.randomUUID().toString());
+ for (OperationDirectories dir : OperationDirectories.values()) {
+ if (dir == OperationDirectories.WitnessDir) {
+ oldDir = new Path(getNewPathString(oldDir.toString(),
+ TESTDIRSTRING, WITNESSDIRSTRING));
+ newDir = new Path(getNewPathString(newDir.toString(),
+ TESTDIRSTRING, WITNESSDIRSTRING));
+ }
+ hdfs.rename(oldDir, newDir, Options.Rename.OVERWRITE);
+ assertTrue("Target directory exists", hdfs.exists(newDir));
+ assertFalse("Source directory does not exist",
+ hdfs.exists(oldDir));
+
+ if (dir == OperationDirectories.TestDir) {
+ snapshottableDirectories.remove(oldDir);
+ snapshottableDirectories.add(newDir);
+ }
+ LOG.info("Renamed directory:" + oldDir + " to directory: " + newDir);
+ numberDirectoryRenamed++;
+ }
+ }
+ }
+ }
+
+ /* Create a new snapshot. */
+ private void createSnapshot() throws IOException {
+ if (snapshottableDirectories.size() > 0) {
+ int index = generator.nextInt(snapshottableDirectories.size());
+ Path randomDir = snapshottableDirectories.get(index);
+ String snapshotName = Integer.toString(generator.nextInt()) + ".ss";
+ hdfs.createSnapshot(randomDir, snapshotName);
+ LOG.info("createSnapshot, directory: " + randomDir +
+ ", snapshot name: " + snapshotName);
+ numberSnapshotCreated++;
+
+ if (pathToSnapshotsMap.containsKey(randomDir)) {
+ pathToSnapshotsMap.get(randomDir).add(snapshotName);
+ } else {
+ pathToSnapshotsMap.put(randomDir,
+ new ArrayList<String>(Arrays.asList(snapshotName)));
+ }
+ }
+ }
+
+ /* Delete an existing snapshot. */
+ private void deleteSnapshot() throws IOException {
+ if (!pathToSnapshotsMap.isEmpty()) {
+ int index = generator.nextInt(pathToSnapshotsMap.size());
+ Object[] snapshotPaths = pathToSnapshotsMap.keySet().toArray();
+ Path snapshotPath = (Path)snapshotPaths[index];
+ ArrayList<String> snapshotNameList=pathToSnapshotsMap.get(snapshotPath);
+
+ String snapshotNameToBeDeleted = snapshotNameList.get(
+ generator.nextInt(snapshotNameList.size()));
+ hdfs.deleteSnapshot(snapshotPath, snapshotNameToBeDeleted);
+ LOG.info("deleteSnapshot, directory: " + snapshotPath +
+ ", snapshot name: " + snapshotNameToBeDeleted);
+ numberSnapshotDeleted++;
+
+ // Adjust pathToSnapshotsMap after snapshot deletion
+ if (snapshotNameList.size() == 1) {
+ pathToSnapshotsMap.remove(snapshotPath);
+ } else {
+ pathToSnapshotsMap.get(snapshotPath).remove(snapshotNameToBeDeleted);
+ }
+ }
+ }
+
+ /* Rename an existing snapshot. */
+ private void renameSnapshot() throws IOException {
+ if (!pathToSnapshotsMap.isEmpty()) {
+ int index = generator.nextInt(pathToSnapshotsMap.size());
+ Object[] snapshotPaths = pathToSnapshotsMap.keySet().toArray();
+ Path snapshotPath = (Path)snapshotPaths[index];
+ ArrayList<String> snapshotNameList =
+ pathToSnapshotsMap.get(snapshotPath);
+
+ String snapshotOldName = snapshotNameList.get(
+ generator.nextInt(snapshotNameList.size()));
+
+ String snapshotOldNameNoExt = snapshotOldName.substring(0,
+ snapshotOldName.lastIndexOf('.')-1);
+
+ String snapshotNewName = snapshotOldNameNoExt + "_rename.ss";
+ hdfs.renameSnapshot(snapshotPath, snapshotOldName, snapshotNewName);
+ LOG.info("renameSnapshot, directory:" + snapshotPath + ", snapshot name:"
+ + snapshotOldName + " to " + snapshotNewName);
+ numberSnapshotRenamed++;
+
+ // Adjust pathToSnapshotsMap after snapshot deletion
+ pathToSnapshotsMap.get(snapshotPath).remove(snapshotOldName);
+ pathToSnapshotsMap.get(snapshotPath).add(snapshotNewName);
+ }
+ }
+
+ /* Create a new test file. */
+ private void createTestFile() throws IOException {
+ if (snapshottableDirectories.size() > 0) {
+ int index = generator.nextInt(snapshottableDirectories.size());
+ Path randomDir = snapshottableDirectories.get(index);
+ if (!randomDir.isRoot()) {
+ randomDir = randomDir.getParent();
+ }
+
+ Path newFile = new Path(randomDir, "createTestFile.log");
+ for (OperationDirectories dir : OperationDirectories.values()) {
+ if (dir == OperationDirectories.WitnessDir) {
+ newFile = new Path(getNewPathString(newFile.toString(),
+ TESTDIRSTRING, WITNESSDIRSTRING));
+ }
+ hdfs.createNewFile(newFile);
+ assertTrue("File exists", hdfs.exists(newFile));
+ LOG.info("createTestFile, file created: " + newFile);
+ numberFileCreated++;
+ }
+ }
+ }
+
+ /* Delete an existing test file. */
+ private void deleteTestFile() throws IOException {
+ if (snapshottableDirectories.size() > 0) {
+ int index = generator.nextInt(snapshottableDirectories.size());
+ Path randomDir = snapshottableDirectories.get(index);
+
+ FileStatus[] fileStatusList = hdfs.listStatus(randomDir);
+ for (FileStatus fsEntry : fileStatusList) {
+ if (fsEntry.isFile()) {
+ Path deleteFile = fsEntry.getPath();
+ for (OperationDirectories dir : OperationDirectories.values()) {
+ if (dir == OperationDirectories.WitnessDir) {
+ deleteFile = new Path(getNewPathString(deleteFile.toString(),
+ TESTDIRSTRING, WITNESSDIRSTRING));
+ }
+ hdfs.delete(deleteFile, false);
+ assertFalse("File does not exists",
+ hdfs.exists(deleteFile));
+ LOG.info("deleteTestFile, file deleted: " + deleteFile);
+ numberFileDeleted++;
+ }
+ break;
+ }
+ }
+ }
+ }
+
+ /* Rename an existing test file. */
+ private void renameTestFile() throws IOException {
+ if (snapshottableDirectories.size() > 0) {
+ int index = generator.nextInt(snapshottableDirectories.size());
+ Path randomDir = snapshottableDirectories.get(index);
+
+ FileStatus[] fileStatusList = hdfs.listStatus(randomDir);
+ for (FileStatus fsEntry : fileStatusList) {
+ if (fsEntry.isFile()) {
+ Path oldFile = fsEntry.getPath();
+ Path newFile = oldFile.suffix("_renameFile");
+ for (OperationDirectories dir : OperationDirectories.values()) {
+ if (dir == OperationDirectories.WitnessDir) {
+ oldFile = new Path(getNewPathString(oldFile.toString(),
+ TESTDIRSTRING, WITNESSDIRSTRING));
+ newFile = new Path(getNewPathString(newFile.toString(),
+ TESTDIRSTRING, WITNESSDIRSTRING));
+ }
+
+ hdfs.rename(oldFile, newFile, Options.Rename.OVERWRITE);
+ assertTrue("Target file exists", hdfs.exists(newFile));
+ assertFalse("Source file does not exist", hdfs.exists(oldFile));
+ LOG.info("Renamed file: " + oldFile + " to file: " + newFile);
+ numberFileRenamed++;
+ }
+ break;
+ }
+ }
+ }
+ }
+
+ /* Check cluster health. */
+ private void checkClusterHealth() throws IOException, InterruptedException,
+ TimeoutException {
+ // 1. FileStatus comparison between test and witness directory
+ FileStatus[] testDirStatus = hdfs.listStatus(TESTDIR);
+ FileStatus[] witnessDirStatus = hdfs.listStatus(WITNESSDIR);
+ assertEquals(witnessDirStatus.length, testDirStatus.length);
+ LOG.info("checkClusterHealth, number of entries verified.");
+
+ Arrays.sort(testDirStatus);
+ Arrays.sort(witnessDirStatus);
+ for (int i = 0; i < testDirStatus.length; i++) {
+ assertEquals(witnessDirStatus[i].getPermission(),
+ testDirStatus[i].getPermission());
+ assertEquals(witnessDirStatus[i].getOwner(),
+ testDirStatus[i].getOwner());
+ assertEquals(witnessDirStatus[i].getGroup(),
+ testDirStatus[i].getGroup());
+ assertEquals(witnessDirStatus[i].getLen(), testDirStatus[i].getLen());
+ assertEquals(witnessDirStatus[i].getBlockSize(),
+ testDirStatus[i].getBlockSize());
+ assertEquals(witnessDirStatus[i].hasAcl(), testDirStatus[i].hasAcl());
+ assertEquals(witnessDirStatus[i].isEncrypted(),
+ testDirStatus[i].isEncrypted());
+ assertEquals(witnessDirStatus[i].isErasureCoded(),
+ testDirStatus[i].isErasureCoded());
+ assertEquals(witnessDirStatus[i].isDirectory(),
+ testDirStatus[i].isDirectory());
+ assertEquals(witnessDirStatus[i].isFile(), testDirStatus[i].isFile());
+ }
+ LOG.info("checkClusterHealth, metadata verified.");
+
+ // Randomly decide whether we want to do a check point
+ if (generator.nextBoolean()) {
+ LOG.info("checkClusterHealth, doing a checkpoint on NN.");
+ hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
+ hdfs.saveNamespace();
+ hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
+ }
+
+ /** Restart name node making sure loading from image successfully */
+ LOG.info("checkClusterHealth, restarting NN.");
+ cluster.restartNameNodes();
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return !cluster.getNameNode().isInSafeMode();
+ }
+ }, 10, 100000);
+
+ assertTrue("NameNode is up", cluster.getNameNode().isActiveState());
+ assertTrue("DataNode is up and running", cluster.isDataNodeUp());
+ assertTrue("Cluster is up and running", cluster.isClusterUp());
+ LOG.info("checkClusterHealth, cluster is healthy.");
+
+ printOperationStats();
+ }
+
+ /* Helper: create a file with a length of filelen. */
+ private void createFile(final String fileName, final long filelen,
+ boolean enableSnapshot) throws IOException {
+ FileSystem fs = cluster.getFileSystem();
+ Path filePath = new Path(fileName);
+ DFSTestUtil.createFile(fs, filePath, filelen, (short) 1, 0);
+
+ // Randomly allow snapshot on parent directly
+ if (enableSnapshot && generator.nextBoolean()) {
+ try {
+ hdfs.allowSnapshot(filePath.getParent());
+ } catch (SnapshotException e) {
+ LOG.info("createFile, exception setting snapshotable directory: " +
+ e.getMessage());
+ }
+ }
+ }
+
+ /* Helper: create a large number of directories and files. */
+ private void createFiles(String rootDir, int fileLength) throws IOException {
+ if (!rootDir.endsWith("/")) {
+ rootDir += "/";
+ }
+
+ // Create files in a directory with random depth, ranging from 0-10.
+ for (int i = 0; i < TOTAL_FILECOUNT; i++) {
+ String filename = rootDir;
+ int dirs = generator.nextInt(MAX_NUM_SUB_DIRECTORIES_LEVEL);
+
+ for (int j=i; j >=(i-dirs); j--) {
+ filename += j + "/";
+ }
+ filename += "file" + i;
+ createFile(filename, fileLength, true);
+ assertTrue("Test file created", hdfs.exists(new Path(filename)));
+ LOG.info("createFiles, file: " + filename + "was created");
+
+ String witnessFile =
+ filename.replaceAll(TESTDIRSTRING, WITNESSDIRSTRING);
+ createFile(witnessFile, fileLength, false);
+ assertTrue("Witness file exists",
+ hdfs.exists(new Path(witnessFile)));
+ LOG.info("createFiles, file: " + witnessFile + "was created");
+ }
+ }
+
+ /* Helper: replace all target string with replacement string in original
+ * string. */
+ private String getNewPathString(String originalString, String targetString,
+ String replacementString) {
+ String str = originalString.replaceAll(targetString, replacementString);
+ LOG.info("Original string: " + originalString);
+ LOG.info("New string: " + str);
+ return str;
+ }
+
+ private void printOperationStats() {
+ LOG.info("Operation statistics for this iteration: ");
+
+ LOG.info("Number of files created: " + numberFileCreated);
+ LOG.info("Number of files deleted: " + numberFileDeleted);
+ LOG.info("Number of files renamed: " + numberFileRenamed);
+
+ LOG.info("Number of directories created: " + numberDirectoryCreated);
+ LOG.info("Number of directories deleted: " + numberDirectoryDeleted);
+ LOG.info("Number of directories renamed: " + numberDirectoryRenamed);
+
+ LOG.info("Number of snapshots created: " + numberSnapshotCreated);
+ LOG.info("Number of snapshots deleted: " + numberSnapshotDeleted);
+ LOG.info("Number of snapshots renamed: " + numberSnapshotRenamed);
+
+ numberFileCreated = 0;
+ numberFileDeleted = 0;
+ numberFileRenamed = 0;
+
+ numberDirectoryCreated = 0;
+ numberDirectoryDeleted = 0;
+ numberDirectoryRenamed = 0;
+
+ numberSnapshotCreated = 0;
+ numberSnapshotDeleted = 0;
+ numberSnapshotRenamed = 0;
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org