You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/10/05 22:06:43 UTC
[14/50] [abbrv] hadoop git commit: HADOOP-15621 2/2 S3Guard:
Implement time-based (TTL) expiry for Authoritative Directory Listing.
Contributed by Gabor Bota
HADOOP-15621 2/2 S3Guard: Implement time-based (TTL) expiry for Authoritative Directory Listing. Contributed by Gabor Bota
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4f752d44
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4f752d44
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4f752d44
Branch: refs/heads/HDFS-12943
Commit: 4f752d442b437b3d297232cfdc8b04aa55c53c5e
Parents: 046b876
Author: Aaron Fabbri <fa...@apache.org>
Authored: Wed Oct 3 00:24:29 2018 -0700
Committer: Aaron Fabbri <fa...@apache.org>
Committed: Wed Oct 3 00:24:29 2018 -0700
----------------------------------------------------------------------
.../fs/s3a/s3guard/ExpirableMetadata.java | 39 +++++++
.../apache/hadoop/fs/s3a/ITestS3GuardTtl.java | 105 +++++++++++++++++++
2 files changed, 144 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f752d44/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/ExpirableMetadata.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/ExpirableMetadata.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/ExpirableMetadata.java
new file mode 100644
index 0000000..5776bd4
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/ExpirableMetadata.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.s3guard;
+
+/**
+ * Expirable Metadata abstract class is for storing the field needed for
+ * metadata classes in S3Guard that could be expired with TTL.
+ */
+public abstract class ExpirableMetadata {
+ private long lastUpdated = 0;
+
+ public long getLastUpdated() {
+ return lastUpdated;
+ }
+
+ public void setLastUpdated(long lastUpdated) {
+ this.lastUpdated = lastUpdated;
+ }
+
+ public boolean isExpired(long ttl, long now) {
+ return (lastUpdated + ttl) <= now;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f752d44/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardTtl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardTtl.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardTtl.java
new file mode 100644
index 0000000..d24009c
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardTtl.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.s3guard.DirListingMetadata;
+import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
+import org.apache.hadoop.fs.s3a.s3guard.S3Guard;
+import org.junit.Assume;
+import org.junit.Test;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.touch;
+import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_AUTHORITATIVE;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.isMetadataStoreAuthoritative;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.metadataStorePersistsAuthoritativeBit;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * These tests are testing the S3Guard TTL (time to live) features.
+ */
+public class ITestS3GuardTtl extends AbstractS3ATestBase {
+
+ /**
+ * Patch the configuration - this test needs disabled filesystem caching.
+ * These tests modify the fs instance that would cause flaky tests.
+ * @return a configuration
+ */
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration configuration = super.createConfiguration();
+ S3ATestUtils.disableFilesystemCaching(configuration);
+ return S3ATestUtils.prepareTestConfiguration(configuration);
+ }
+
+ @Test
+ public void testDirectoryListingAuthoritativeTtl() throws Exception {
+
+ final S3AFileSystem fs = getFileSystem();
+ Assume.assumeTrue(fs.hasMetadataStore());
+ final MetadataStore ms = fs.getMetadataStore();
+
+ Assume.assumeTrue("MetadataStore should be capable for authoritative "
+ + "storage of directories to run this test.",
+ metadataStorePersistsAuthoritativeBit(ms));
+
+ Assume.assumeTrue("MetadataStore should be authoritative for this test",
+ isMetadataStoreAuthoritative(getFileSystem().getConf()));
+
+ S3Guard.ITtlTimeProvider mockTimeProvider =
+ mock(S3Guard.ITtlTimeProvider.class);
+ S3Guard.ITtlTimeProvider restoreTimeProvider = fs.getTtlTimeProvider();
+ fs.setTtlTimeProvider(mockTimeProvider);
+ when(mockTimeProvider.getNow()).thenReturn(100L);
+ when(mockTimeProvider.getAuthoritativeDirTtl()).thenReturn(1L);
+
+ Path dir = path("ttl/");
+ Path file = path("ttl/afile");
+
+ try {
+ fs.mkdirs(dir);
+ touch(fs, file);
+
+ // get an authoritative listing in ms
+ fs.listStatus(dir);
+ // check if authoritative
+ DirListingMetadata dirListing =
+ S3Guard.listChildrenWithTtl(ms, dir, mockTimeProvider);
+ assertTrue("Listing should be authoritative.",
+ dirListing.isAuthoritative());
+ // change the time, and assume it's not authoritative anymore
+ when(mockTimeProvider.getNow()).thenReturn(102L);
+ dirListing = S3Guard.listChildrenWithTtl(ms, dir, mockTimeProvider);
+ assertFalse("Listing should not be authoritative.",
+ dirListing.isAuthoritative());
+
+ // get an authoritative listing in ms again - retain test
+ fs.listStatus(dir);
+ // check if authoritative
+ dirListing = S3Guard.listChildrenWithTtl(ms, dir, mockTimeProvider);
+ assertTrue("Listing shoud be authoritative after listStatus.",
+ dirListing.isAuthoritative());
+ } finally {
+ fs.delete(dir, true);
+ fs.setTtlTimeProvider(restoreTimeProvider);
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org