You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ma...@apache.org on 2017/05/25 12:13:17 UTC
[5/6] hadoop git commit: Revert "HADOOP-13760. S3Guard: add delete
tracking."
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1775960/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java
deleted file mode 100644
index 43eb2c0..0000000
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Set;
-
-/**
- * Place for the S3A listing classes; keeps all the small classes under control.
- */
-public class TestListing extends AbstractS3AMockTest {
-
- private static class MockRemoteIterator<FileStatus> implements
- RemoteIterator<FileStatus> {
- private Iterator<FileStatus> iterator;
-
- MockRemoteIterator(Collection<FileStatus> source) {
- iterator = source.iterator();
- }
-
- public boolean hasNext() {
- return iterator.hasNext();
- }
-
- public FileStatus next() {
- return iterator.next();
- }
- }
-
- private FileStatus blankFileStatus(Path path) {
- return new FileStatus(0, true, 0, 0, 0, path);
- }
-
- @Test
- public void testTombstoneReconcilingIterator() throws Exception {
- Path parent = new Path("/parent");
- Path liveChild = new Path(parent, "/liveChild");
- Path deletedChild = new Path(parent, "/deletedChild");
- Path[] allFiles = {parent, liveChild, deletedChild};
- Path[] liveFiles = {parent, liveChild};
-
- Listing listing = new Listing(fs);
- Collection<FileStatus> statuses = new ArrayList<>();
- statuses.add(blankFileStatus(parent));
- statuses.add(blankFileStatus(liveChild));
- statuses.add(blankFileStatus(deletedChild));
-
- Set<Path> tombstones = new HashSet<>();
- tombstones.add(deletedChild);
-
- RemoteIterator<FileStatus> sourceIterator = new MockRemoteIterator(
- statuses);
- RemoteIterator<LocatedFileStatus> locatedIterator =
- listing.createLocatedFileStatusIterator(sourceIterator);
- RemoteIterator<LocatedFileStatus> reconcilingIterator =
- listing.createTombstoneReconcilingIterator(locatedIterator, tombstones);
-
- Set<Path> expectedPaths = new HashSet<>();
- expectedPaths.add(parent);
- expectedPaths.add(liveChild);
-
- Set<Path> actualPaths = new HashSet<>();
- while (reconcilingIterator.hasNext()) {
- actualPaths.add(reconcilingIterator.next().getPath());
- }
- Assert.assertTrue(actualPaths.equals(expectedPaths));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1775960/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
index dfa8a9e..99acf6e 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.fs.s3a.s3guard;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.s3a.S3ATestUtils;
import org.apache.hadoop.fs.s3a.Tristate;
@@ -135,52 +134,12 @@ public abstract class MetadataStoreTestBase extends Assert {
}
/**
- * Helper function for verifying DescendantsIterator and
- * MetadataStoreListFilesIterator behavior.
- * @param createNodes List of paths to create
- * @param checkNodes List of paths that the iterator should return
- * @throws IOException
- */
- private void doTestDescendantsIterator(
- Class implementation, String[] createNodes,
- String[] checkNodes) throws Exception {
- // we set up the example file system tree in metadata store
- for (String pathStr : createNodes) {
- final FileStatus status = pathStr.contains("file")
- ? basicFileStatus(strToPath(pathStr), 100, false)
- : basicFileStatus(strToPath(pathStr), 0, true);
- ms.put(new PathMetadata(status));
- }
-
- final PathMetadata rootMeta = new PathMetadata(makeDirStatus("/"));
- RemoteIterator<FileStatus> iterator;
- if (implementation == DescendantsIterator.class) {
- iterator = new DescendantsIterator(ms, rootMeta);
- } else if (implementation == MetadataStoreListFilesIterator.class) {
- iterator = new MetadataStoreListFilesIterator(ms, rootMeta, false);
- } else {
- throw new UnsupportedOperationException("Unrecognized class");
- }
-
- final Set<String> actual = new HashSet<>();
- while (iterator.hasNext()) {
- final Path p = iterator.next().getPath();
- actual.add(Path.getPathWithoutSchemeAndAuthority(p).toString());
- }
- LOG.info("We got {} by iterating DescendantsIterator", actual);
-
- if (!allowMissing()) {
- assertEquals(Sets.newHashSet(checkNodes), actual);
- }
- }
-
- /**
* Test that we can get the whole sub-tree by iterating DescendantsIterator.
*
* The tree is similar to or same as the example in code comment.
*/
@Test
- public void testDescendantsIterator() throws Exception {
+ public void testDescendantsIterator() throws IOException {
final String[] tree = new String[] {
"/dir1",
"/dir1/dir2",
@@ -193,38 +152,26 @@ public abstract class MetadataStoreTestBase extends Assert {
"/dir1/dir3/dir5/file4",
"/dir1/dir3/dir6"
};
- doTestDescendantsIterator(DescendantsIterator.class,
- tree, tree);
- }
+ // we set up the example file system tree in metadata store
+ for (String pathStr : tree) {
+ final FileStatus status = pathStr.contains("file")
+ ? basicFileStatus(strToPath(pathStr), 100, false)
+ : basicFileStatus(strToPath(pathStr), 0, true);
+ ms.put(new PathMetadata(status));
+ }
- /**
- * Test that we can get the correct subset of the tree with
- * MetadataStoreListFilesIterator.
- *
- * The tree is similar to or same as the example in code comment.
- */
- @Test
- public void testMetadataStoreListFilesIterator() throws Exception {
- final String[] wholeTree = new String[] {
- "/dir1",
- "/dir1/dir2",
- "/dir1/dir3",
- "/dir1/dir2/file1",
- "/dir1/dir2/file2",
- "/dir1/dir3/dir4",
- "/dir1/dir3/dir5",
- "/dir1/dir3/dir4/file3",
- "/dir1/dir3/dir5/file4",
- "/dir1/dir3/dir6"
- };
- final String[] leafNodes = new String[] {
- "/dir1/dir2/file1",
- "/dir1/dir2/file2",
- "/dir1/dir3/dir4/file3",
- "/dir1/dir3/dir5/file4"
- };
- doTestDescendantsIterator(MetadataStoreListFilesIterator.class, wholeTree,
- leafNodes);
+ final Set<String> actual = new HashSet<>();
+ final PathMetadata rootMeta = new PathMetadata(makeDirStatus("/"));
+ for (DescendantsIterator desc = new DescendantsIterator(ms, rootMeta);
+ desc.hasNext();) {
+ final Path p = desc.next().getPath();
+ actual.add(Path.getPathWithoutSchemeAndAuthority(p).toString());
+ }
+ LOG.info("We got {} by iterating DescendantsIterator", actual);
+
+ if (!allowMissing()) {
+ assertEquals(Sets.newHashSet(tree), actual);
+ }
}
@Test
@@ -311,7 +258,7 @@ public abstract class MetadataStoreTestBase extends Assert {
/* Ensure delete happened. */
assertDirectorySize("/ADirectory1/db1", 1);
PathMetadata meta = ms.get(strToPath("/ADirectory1/db1/file2"));
- assertTrue("File deleted", meta == null || meta.isDeleted());
+ assertNull("File deleted", meta);
}
@Test
@@ -337,10 +284,10 @@ public abstract class MetadataStoreTestBase extends Assert {
ms.deleteSubtree(strToPath(p + "/ADirectory1/db1/"));
assertEmptyDirectory(p + "/ADirectory1");
- assertDeleted(p + "/ADirectory1/db1");
- assertDeleted(p + "/ADirectory1/file1");
- assertDeleted(p + "/ADirectory1/file2");
- assertDeleted(p + "/ADirectory1/db1/dc1/dd1/deepFile");
+ assertNotCached(p + "/ADirectory1/db1");
+ assertNotCached(p + "/ADirectory1/file1");
+ assertNotCached(p + "/ADirectory1/file2");
+ assertNotCached(p + "/ADirectory1/db1/dc1/dd1/deepFile");
assertEmptyDirectory(p + "/ADirectory2");
}
@@ -355,11 +302,11 @@ public abstract class MetadataStoreTestBase extends Assert {
setUpDeleteTest();
ms.deleteSubtree(strToPath("/"));
- assertDeleted("/ADirectory1");
- assertDeleted("/ADirectory2");
- assertDeleted("/ADirectory2/db1");
- assertDeleted("/ADirectory2/db1/file1");
- assertDeleted("/ADirectory2/db1/file2");
+ assertNotCached("/ADirectory1");
+ assertNotCached("/ADirectory2");
+ assertNotCached("/ADirectory2/db1");
+ assertNotCached("/ADirectory2/db1/file1");
+ assertNotCached("/ADirectory2/db1/file2");
}
@Test
@@ -403,12 +350,6 @@ public abstract class MetadataStoreTestBase extends Assert {
verifyFileStatus(meta.getFileStatus(), 100);
}
- if (!(ms instanceof NullMetadataStore)) {
- ms.delete(strToPath(filePath));
- meta = ms.get(strToPath(filePath));
- assertTrue("Tombstone not left for deleted file", meta.isDeleted());
- }
-
meta = ms.get(strToPath(dirPath));
if (!allowMissing() || meta != null) {
assertNotNull("Get found file (dir)", meta);
@@ -500,7 +441,6 @@ public abstract class MetadataStoreTestBase extends Assert {
dirMeta = ms.listChildren(strToPath("/a1"));
if (!allowMissing() || dirMeta != null) {
- dirMeta = dirMeta.withoutTombstones();
assertListingsEqual(dirMeta.getListing(), "/a1/b1", "/a1/b2");
}
@@ -546,7 +486,6 @@ public abstract class MetadataStoreTestBase extends Assert {
Collection<PathMetadata> entries;
DirListingMetadata dirMeta = ms.listChildren(strToPath("/"));
if (!allowMissing() || dirMeta != null) {
- dirMeta = dirMeta.withoutTombstones();
assertNotNull("Listing root", dirMeta);
entries = dirMeta.getListing();
assertListingsEqual(entries, "/a1", "/a2", "/a3");
@@ -574,12 +513,13 @@ public abstract class MetadataStoreTestBase extends Assert {
dirMeta = ms.listChildren(strToPath("/a1"));
if (!allowMissing() || dirMeta != null) {
assertNotNull("Listing /a1", dirMeta);
- entries = dirMeta.withoutTombstones().getListing();
+ entries = dirMeta.getListing();
assertListingsEqual(entries, "/a1/b2");
}
PathMetadata meta = ms.get(strToPath("/a1/b1/file1"));
- assertTrue("Src path deleted", meta == null || meta.isDeleted());
+ // TODO allow return of PathMetadata with isDeleted == true
+ assertNull("Src path deleted", meta);
// Assert dest looks right
meta = ms.get(strToPath("/b1/file1"));
@@ -656,7 +596,7 @@ public abstract class MetadataStoreTestBase extends Assert {
ms.prune(cutoff);
ls = ms.listChildren(strToPath("/pruneFiles"));
if (allowMissing()) {
- assertDeleted("/pruneFiles/old");
+ assertNotCached("/pruneFiles/old");
} else {
assertListingsEqual(ls.getListing(), "/pruneFiles/new");
}
@@ -685,7 +625,7 @@ public abstract class MetadataStoreTestBase extends Assert {
ms.prune(cutoff);
- assertDeleted("/pruneDirs/dir/file");
+ assertNotCached("/pruneDirs/dir/file");
}
/*
@@ -706,7 +646,6 @@ public abstract class MetadataStoreTestBase extends Assert {
"file3"));
DirListingMetadata dirMeta = ms.listChildren(strToPath(parent));
if (!allowMissing() || dirMeta != null) {
- dirMeta = dirMeta.withoutTombstones();
assertNotNull("list after putListStatus", dirMeta);
Collection<PathMetadata> entries = dirMeta.getListing();
assertNotNull("listStatus has entries", entries);
@@ -761,7 +700,6 @@ public abstract class MetadataStoreTestBase extends Assert {
assertNotNull("Directory " + pathStr + " in cache", dirMeta);
}
if (!allowMissing() || dirMeta != null) {
- dirMeta = dirMeta.withoutTombstones();
assertEquals("Number of entries in dir " + pathStr, size,
nonDeleted(dirMeta.getListing()).size());
}
@@ -770,27 +708,21 @@ public abstract class MetadataStoreTestBase extends Assert {
/** @return only file statuses which are *not* marked deleted. */
private Collection<PathMetadata> nonDeleted(
Collection<PathMetadata> statuses) {
- Collection<PathMetadata> currentStatuses = new ArrayList<>();
- for (PathMetadata status : statuses) {
- if (!status.isDeleted()) {
- currentStatuses.add(status);
- }
- }
- return currentStatuses;
+ /* TODO: filter out paths marked for deletion. */
+ return statuses;
}
- private void assertDeleted(String pathStr) throws IOException {
+ private void assertNotCached(String pathStr) throws IOException {
+ // TODO this should return an entry with deleted flag set
Path path = strToPath(pathStr);
PathMetadata meta = ms.get(path);
- boolean cached = meta != null && !meta.isDeleted();
- assertFalse(pathStr + " should not be cached.", cached);
+ assertNull(pathStr + " should not be cached.", meta);
}
protected void assertCached(String pathStr) throws IOException {
Path path = strToPath(pathStr);
PathMetadata meta = ms.get(path);
- boolean cached = meta != null && !meta.isDeleted();
- assertTrue(pathStr + " should be cached.", cached);
+ assertNotNull(pathStr + " should be cached.", meta);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1775960/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
index 27416bb..3584b54 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
@@ -294,7 +294,7 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
if (oldMetas != null) {
// put all metadata of old paths and verify
ms.put(new DirListingMetadata(oldDir, oldMetas, false));
- assertEquals(0, ms.listChildren(newDir).withoutTombstones().numEntries());
+ assertEquals(0, ms.listChildren(newDir).numEntries());
assertTrue(CollectionUtils.isEqualCollection(oldMetas,
ms.listChildren(oldDir).getListing()));
@@ -306,7 +306,7 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
// move the old paths to new paths and verify
ms.move(pathsToDelete, newMetas);
- assertEquals(0, ms.listChildren(oldDir).withoutTombstones().numEntries());
+ assertEquals(0, ms.listChildren(oldDir).numEntries());
if (newMetas != null) {
assertTrue(CollectionUtils.isEqualCollection(newMetas,
ms.listChildren(newDir).getListing()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1775960/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
index 89d0498..4cffc6f 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
@@ -75,7 +75,7 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
@Test
public void testClearByAncestor() {
- Map<Path, PathMetadata> map = new HashMap<>();
+ Map<Path, String> map = new HashMap<>();
// 1. Test paths without scheme/host
assertClearResult(map, "", "/", 0);
@@ -90,37 +90,21 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
assertClearResult(map, p, "/invalid", 5);
}
- private static void populateMap(Map<Path, PathMetadata> map,
- String prefix) {
- populateEntry(map, new Path(prefix + "/dirA/dirB/"));
- populateEntry(map, new Path(prefix + "/dirA/dirB/dirC"));
- populateEntry(map, new Path(prefix + "/dirA/dirB/dirC/file1"));
- populateEntry(map, new Path(prefix + "/dirA/dirB/dirC/file2"));
- populateEntry(map, new Path(prefix + "/dirA/file1"));
+ private static void populateMap(Map<Path, String> map, String prefix) {
+ String dummyVal = "dummy";
+ map.put(new Path(prefix + "/dirA/dirB/"), dummyVal);
+ map.put(new Path(prefix + "/dirA/dirB/dirC"), dummyVal);
+ map.put(new Path(prefix + "/dirA/dirB/dirC/file1"), dummyVal);
+ map.put(new Path(prefix + "/dirA/dirB/dirC/file2"), dummyVal);
+ map.put(new Path(prefix + "/dirA/file1"), dummyVal);
}
- private static void populateEntry(Map<Path, PathMetadata> map,
- Path path) {
- map.put(path, new PathMetadata(new FileStatus(0, true, 0, 0, 0, path)));
- }
-
- private static int sizeOfMap(Map<Path, PathMetadata> map) {
- int count = 0;
- for (PathMetadata meta : map.values()) {
- if (!meta.isDeleted()) {
- count++;
- }
- }
- return count;
- }
-
- private static void assertClearResult(Map <Path, PathMetadata> map,
+ private static void assertClearResult(Map <Path, String> map,
String prefixStr, String pathStr, int leftoverSize) {
populateMap(map, prefixStr);
- LocalMetadataStore.deleteHashByAncestor(new Path(prefixStr + pathStr), map,
- true);
+ LocalMetadataStore.clearHashByAncestor(new Path(prefixStr + pathStr), map);
assertEquals(String.format("Map should have %d entries", leftoverSize),
- leftoverSize, sizeOfMap(map));
+ leftoverSize, map.size());
map.clear();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1775960/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractITestS3AMetadataStoreScale.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractITestS3AMetadataStoreScale.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractITestS3AMetadataStoreScale.java
index 876cc80..9b8e3c1 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractITestS3AMetadataStoreScale.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractITestS3AMetadataStoreScale.java
@@ -114,7 +114,7 @@ public abstract class AbstractITestS3AMetadataStoreScale extends
describe("Running move workload");
NanoTimer moveTimer = new NanoTimer();
LOG.info("Running {} moves of {} paths each", operations,
- origMetas.size());
+ origPaths.size());
for (int i = 0; i < operations; i++) {
Collection<Path> toDelete;
Collection<PathMetadata> toCreate;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org