You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sa...@apache.org on 2026/07/13 09:39:08 UTC
(ozone) branch HDDS-8342 updated: HDDS-15447. Persist bucket scanned key pointer periodically (#10412)
This is an automated email from the ASF dual-hosted git repository.
ChenSammi pushed a commit to branch HDDS-8342
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-8342 by this push:
new daef2f19c29 HDDS-15447. Persist bucket scanned key pointer periodically (#10412)
daef2f19c29 is described below
commit daef2f19c2946ad8e909323f5956b1884bba64fe
Author: Sammi Chen <sa...@apache.org>
AuthorDate: Mon Jul 13 17:39:01 2026 +0800
HDDS-15447. Persist bucket scanned key pointer periodically (#10412)
---
.../common/src/main/resources/ozone-default.xml | 15 +
.../docs/content/design/lifecycle-task-resume.md | 80 ++
hadoop-hdds/docs/content/feature/Lifecycle.md | 11 +-
hadoop-hdds/docs/content/feature/Lifecycle.zh.md | 11 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 2 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 7 +
.../apache/hadoop/ozone/om/helpers/OmLCRule.java | 13 +-
.../ozone/om/helpers/OmLifecycleScanState.java | 255 ++++
.../hadoop/ozone/om/helpers/TestOmLCRule.java | 33 +
.../ozone/om/helpers/TestOmLifecycleScanState.java | 79 ++
.../src/main/proto/OmClientProtocol.proto | 23 +
.../apache/hadoop/ozone/om/OMMetadataManager.java | 8 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 8 +
.../hadoop/ozone/om/codec/OMDBDefinition.java | 11 +-
.../om/ratis/utils/OzoneManagerRatisUtils.java | 3 +
.../ozone/om/request/key/OMKeysDeleteRequest.java | 38 +-
.../om/request/key/OmKeysDeleteRequestWithFSO.java | 6 +-
.../lifecycle/OMLifecycleSaveScanStateRequest.java | 74 ++
.../om/response/key/OMKeysDeleteResponse.java | 17 +-
.../response/key/OMKeysDeleteResponseWithFSO.java | 15 +-
.../OMLifecycleSaveScanStateResponse.java | 51 +
.../ozone/om/service/KeyLifecycleService.java | 988 +++++++++++----
.../hadoop/ozone/om/TestOmMetadataManager.java | 4 +-
.../TestOMLifecycleSaveScanStateRequest.java | 118 ++
.../om/response/key/TestOMKeysDeleteResponse.java | 2 +-
.../key/TestOMKeysDeleteResponseWithFSO.java | 2 +-
.../TestOMLifecycleSaveScanStateResponse.java | 61 +
.../ozone/om/service/TestKeyLifecycleService.java | 1290 +++++++++++++++++---
28 files changed, 2821 insertions(+), 404 deletions(-)
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index a7c4d56e531..2dbc98adf1e 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -5140,4 +5140,19 @@
lifecycle rules.
</description>
</property>
+ <property>
+ <name>ozone.lifecycle.service.state.save.interval.ms</name>
+ <value>300000</value>
+ <tag>OZONE</tag>
+ <description>The interval of bucket scan task saves its pointer to DB. Default is 5 mins.</description>
+ </property>
+ <property>
+ <name>ozone.lifecycle.service.state.save.keys.processed</name>
+ <value>100000</value>
+ <tag>OZONE</tag>
+ <description>Bucket scan task will save its pointer to DB by default every 100000 keys are scanned. Bucket scan
+ pointer save will happen when either ozone.lifecycle.service.state.save.interval.ms or
+ ozone.lifecycle.service.state.save.keys.processed is satisfied.
+ </description>
+ </property>
</configuration>
diff --git a/hadoop-hdds/docs/content/design/lifecycle-task-resume.md b/hadoop-hdds/docs/content/design/lifecycle-task-resume.md
new file mode 100644
index 00000000000..d800e21d4fe
--- /dev/null
+++ b/hadoop-hdds/docs/content/design/lifecycle-task-resume.md
@@ -0,0 +1,80 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+# Design for Resumable Lifecycle Scans(HDDS-8342)
+
+## Problem Statement:
+
+The `HDDS-8342` branch introduces the `KeyLifecycleService`, a background service running on the Ozone Manager (OM) Leader to enforce bucket lifecycle rules (expiration, moving to trash, and aborting incomplete multipart uploads).
+The entire bucket is scanned in a single `call()` execution. If the OM restarts, crashes, or a leader transfer occurs, the scan state is lost. The new leader must restart the scan from the beginning.
+For buckets with billions of keys, the scan may never complete if leader transfers happen periodically.
+
+## Design: Persisting Bucket Scan Pointers
+
+To solve the resumability issues, we need to persist the scan progress (the "pointer") to the OM DB. This ensures that a new OM leader can resume from where the previous leader left off.
+
+### 2.1 Data Structure for the Scan Pointer
+
+Define a new Protobuf message `LifecycleScanState` to capture the exact position of the scan.
+
+```protobuf
+message LifecycleScanState {
+ optional string bucketKey = 1; // e.g., /volume/bucket
+ optional uint64 bucketObjID = 2; // bucket's object ID, in case the bucket is deleted and recreated with same name
+ optional uint64 lifecycleConfigurationUpdateID = 3; // lifecycle configuration update ID, in case the bucket is updated with new rules
+ optional uint64 scanStartTime = 4; // Epoch time when this full scan started
+ optional uint64 scanEndTime = 5; // Epoch time when this full scan is completed
+ optional string lastScannedKey = 6; // the last scanned key in the bucket(for both OBS and FSO)
+ optional string lastScannedDir = 7; // the last scanned dir path, e.g /dir1/dir2/dir3
+ optional string lastScannedDirKey = 8; // the last scanned dir key in directoryTable, e.g /0/1/3/dir3
+ optional string lastScannedMpuKey = 9;
+}
+```
+
+### OM DB Schema Updates
+Add a new table `lifecycleScanStateTable` to `OMMetadataManager` to store the scan states:
+- **Table Name:** `lifecycleScanStateTable`
+- **Key:** `bucketKey` (String, e.g., `/volumeName/bucketName`)
+- **Value:** `LifecycleScanState`
+
+### When to Persist the Pointer
+Persisting the pointer for every key would overwhelm Ratis and RocksDB. We should checkpoint periodically:
+
+1. **Piggybacking on Deletes:** Add an optional `LifecycleScanState` field to `DeleteKeysRequest`. When the OM state machine applies the deletion, it atomically updates the `lifecycleScanStateTable` with the new pointer. This guarantees exactly-once semantics for the scan pointer relative to deletions.
+2. **Move to trash**: Since there is no `RenameKeysRequest`, rename has be called multiple times for a batch of keys. We introduce a new OM request `SaveLifecycleScanStateRequest`. After a batch of keys are moved to trash, call `SaveLifecycleScanStateRequest` explicitly to persist the state.
+3. **Periodic Standalone Checkpoints:** If no keys are expired (e.g., scanning millions of valid keys), we still need to save progress. The `LifecycleActionTask` will send this request periodically (e.g., every 100,000 keys iterated, or every 1 minute of execution time).
+3. **End of Scan:** When the scan for a bucket finishes, a `SaveLifecycleScanStateRequest` is sent to mark state as completed by recording the completion time.
+
+### How to Resume the Scan
+When `KeyLifecycleService` schedules a `LifecycleActionTask` for a bucket, it first reads the `LifecycleScanState` from the `lifecycleScanStateTable`.
+
+- **OBS/Legacy Resumption:**
+ The iterator for `keyTable` is initialized to seek to `lastScannedKey` instead of the bucket prefix.
+ ```java
+ TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> keyTblItr = keyTable.iterator(bucketPrefix);
+ if (state.getLastScannedKey() != null) {
+ keyTblItr.seek(state.getLastScannedKey());
+ // skip the exact match since it was already processed
+ }
+ ```
+
+- **FSO Resumption:**
+ Since FSO bucket is iterated via a Depth-First Search (DFS) way, any directory that is after the `lastScannedDir` in the traversal path can be skipped.
+
+- **MPU Resumption:**
+ // TODO: implement MPU resumption
+ The `multipartInfoTable` iterator seeks to `lastScannedMpuKey` and continues.
diff --git a/hadoop-hdds/docs/content/feature/Lifecycle.md b/hadoop-hdds/docs/content/feature/Lifecycle.md
index 709bdf51f54..ec2302918bc 100644
--- a/hadoop-hdds/docs/content/feature/Lifecycle.md
+++ b/hadoop-hdds/docs/content/feature/Lifecycle.md
@@ -132,19 +132,22 @@ For FILE_SYSTEM_OPTIMIZED (FSO) buckets, the Prefix must be a normalized and val
- Cannot start with `/`. FSO bucket prefixes are relative to the bucket root and do not need a leading slash.
- Cannot contain consecutive slashes `//`.
- Path components cannot contain `.` (current directory), `..` (parent directory), or `:`.
+- Must end with "/", or "" for root.
The following table shows examples of valid and invalid prefixes:
| Prefix | Valid for FSO Bucket | Reason |
-|--------|----------------------|--------|
+|----|----------------------|--|
| `logs/` | Valid | Normalized directory prefix |
| `data/2024/` | Valid | Multi-level directory prefix |
-| `archive` | Valid | Simple prefix without slash |
+| `archive` | Invalid | Without tailing slash |
| `/logs/` | Invalid | Cannot start with `/`, use `logs/` instead |
| `data//backup/` | Invalid | Contains consecutive slashes `//`, use `data/backup/` instead |
| `data/../secret/` | Invalid | Contains `..`, parent directory references are not allowed |
| `data/./logs/` | Invalid | Contains `.`, current directory references are not allowed |
| `.Trash/` | Invalid | Cannot point to trash directories |
+| `` | Valid | Point to Bucket's root directory |
+| `/` | Invalid | It doesn't point to Bucket's root directory. Use "" instead |
## S3 Gateway API
@@ -354,9 +357,9 @@ ozone admin om lifecycle resume [-id=<omServiceId>] [-host=<omHost>]
In an OM HA deployment, the lifecycle service only runs on the leader OM. When a Transfer Leader operation is performed:
1. The lifecycle evaluation tasks running on the old leader will be interrupted.
-2. After the new leader is elected, the lifecycle service restarts from the beginning. Previously evaluated buckets are not skipped, and the task starts over from the first bucket.
+2. After the new leader is elected, the lifecycle service will skip the previously evaluated buckets/bucket contents, starting from the interrupted bucket.
-Therefore, in scenarios with frequent leader transfers, it is recommended to monitor the actual execution of the lifecycle service to ensure expired objects are cleaned up in a timely manner.
+In scenarios with frequent leader transfers, it is recommended to monitor the actual execution of the lifecycle service to ensure expired objects are cleaned up in a timely manner.
### Impact of Mass Key Expiration on Metadata Performance
diff --git a/hadoop-hdds/docs/content/feature/Lifecycle.zh.md b/hadoop-hdds/docs/content/feature/Lifecycle.zh.md
index b8aaa0f9d3e..cab07a27864 100644
--- a/hadoop-hdds/docs/content/feature/Lifecycle.zh.md
+++ b/hadoop-hdds/docs/content/feature/Lifecycle.zh.md
@@ -131,19 +131,22 @@ FSO Bucket 的额外校验规则:
- 不能以 `/` 开头。FSO Bucket 的 Prefix 是相对于 Bucket 根目录的路径,无需前导斜杠。
- 不能包含连续的斜杠 `//`。
- 路径组件中不能包含 `.`(当前目录)、`..`(父目录)或 `:`。
+- 路径必须以 `/` 结尾, 或者 “” 代表root。
以下是合法与不合法 Prefix 的对照示例:
| Prefix | FSO Bucket 是否合法 | 原因 |
-|--------|---------------------|------|
+|----|--------------------|-----|
| `logs/` | 合法 | 规范化的目录前缀 |
| `data/2024/` | 合法 | 多级目录前缀 |
-| `archive` | 合法 | 无斜杠的简单前缀 |
+| `archive` | 不合法 | 无斜杠结尾 |
| `/logs/` | 不合法 | 不能以 `/` 开头,应使用 `logs/` |
| `data//backup/` | 不合法 | 包含连续斜杠 `//`,应使用 `data/backup/` |
| `data/../secret/` | 不合法 | 包含 `..`,不允许使用父目录引用 |
| `data/./logs/` | 不合法 | 包含 `.`,不允许使用当前目录引用 |
| `.Trash/` | 不合法 | 不能指向回收站目录 |
+| `` | 合法 | 指向Bucket 根目录 |
+| `/` | 不合法 | 它不指向 Bucket 根目录。正确代表根目录的前缀是 “” |
## S3 Gateway API
@@ -353,9 +356,9 @@ ozone admin om lifecycle resume [-id=<omServiceId>] [-host=<omHost>]
在 OM HA 部署中,生命周期服务仅在 Leader OM 上运行。当执行 Transfer Leader 操作时:
1. 旧 Leader 上正在运行的生命周期评估任务会被中断。
-2. 新 Leader 当选后,会重新从头启动生命周期服务,已经评估过的 Bucket 不会被跳过,任务将从第一个 Bucket 重新开始。
+2. 新 Leader 当选后,会重新从头启动生命周期服务,已经评估过的 Bucket 会被跳过,任务将从中断的Bucket 重新开始。
-因此,在频繁进行 Leader 切换的场景下,建议关注生命周期服务的实际执行情况,确保过期对象能被及时清理。
+在频繁进行 Leader 切换的场景下,建议关注生命周期服务的实际执行情况,确保过期对象能被及时清理。
### 大批量 Key 过期对元数据性能的影响
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index 67ea6380fdf..575b8d2aec5 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -327,6 +327,7 @@ public static boolean isReadOnly(OMRequest omRequest) {
case SetLifecycleConfiguration:
case DeleteLifecycleConfiguration:
case SetLifecycleServiceStatus:
+ case SaveLifecycleScanState:
case UnknownCommand:
return false;
case EchoRPC:
@@ -462,6 +463,7 @@ public static boolean shouldSendToFollower(OMRequest omRequest) {
case SetLifecycleConfiguration:
case DeleteLifecycleConfiguration:
case SetLifecycleServiceStatus:
+ case SaveLifecycleScanState:
case UnknownCommand:
return false;
case EchoRPC:
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 7857c28f339..7d2d0e860bd 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -199,6 +199,13 @@ public final class OMConfigKeys {
"ozone.lifecycle.service.delete.cached.directory.max-count";
public static final long OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT_DEFAULT = 1000000;
+ public static final String OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS =
+ "ozone.lifecycle.service.state.save.interval.ms";
+ public static final long OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT = 5 * 60 * 1000;
+ public static final String OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED =
+ "ozone.lifecycle.service.state.save.keys.processed";
+ public static final long OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT = 100000;
+
public static final String OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED =
"ozone.lifecycle.service.move.to.trash.enabled";
public static final boolean
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java
index 10b627790a0..ce9f56c30b8 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.om.helpers;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateAndNormalizePrefix;
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validatePrefixLength;
import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTrashPrefix;
@@ -28,7 +29,6 @@
import net.jcip.annotations.Immutable;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleAction;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleRule;
@@ -61,7 +61,7 @@ private OmLCRule() {
private OmLCRule(Builder builder) {
this.prefix = builder.prefix;
if (this.prefix != null) {
- this.directoryStylePrefix = this.prefix.contains(OzoneConsts.OM_KEY_PREFIX);
+ this.directoryStylePrefix = this.prefix.contains(OM_KEY_PREFIX);
} else {
this.directoryStylePrefix = false;
}
@@ -196,6 +196,14 @@ public void valid(BucketLayout bucketLayout, Long creationTime) throws OMExcepti
OMException.ResultCodes.INVALID_REQUEST);
}
action.valid(creationTime);
+
+ if (bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
+ if (getEffectivePrefix() != null && !getEffectivePrefix().isEmpty() &&
+ !getEffectivePrefix().endsWith(OM_KEY_PREFIX)) {
+ throw new OMException("FILE_SYSTEM_OPTIMIZED bucket prefix must end with '/'.",
+ OMException.ResultCodes.INVALID_REQUEST);
+ }
+ }
}
if (prefix != null && filter != null) {
@@ -323,7 +331,6 @@ public static OmLCRule getFromProtobuf(LifecycleRule lifecycleRule, BucketLayout
if (lifecycleRule.hasFilter()) {
builder.setFilter(OmLCFilter.getFromProtobuf(lifecycleRule.getFilter(), layout));
}
-
return builder.build();
}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleScanState.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleScanState.java
new file mode 100644
index 00000000000..41bc5de16aa
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleScanState.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+import org.apache.hadoop.hdds.utils.db.Codec;
+import org.apache.hadoop.hdds.utils.db.DelegatedCodec;
+import org.apache.hadoop.hdds.utils.db.Proto2Codec;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleScanState;
+
+/**
+ * POJO for LifecycleScanState.
+ */
+public class OmLifecycleScanState {
+ private String bucketKey;
+ private long bucketObjID;
+ private long lifecycleConfigurationUpdateID;
+ private long scanStartTime;
+ private Long scanEndTime;
+ private String lastScannedKey;
+ private String lastScannedDir;
+ private String lastScannedDirKey;
+
+ private static final Codec<OmLifecycleScanState> CODEC = new DelegatedCodec<>(
+ Proto2Codec.get(LifecycleScanState.getDefaultInstance()),
+ OmLifecycleScanState::getFromProtobuf,
+ OmLifecycleScanState::getProtobuf,
+ OmLifecycleScanState.class);
+
+ public static Codec<OmLifecycleScanState> getCodec() {
+ return CODEC;
+ }
+
+ public OmLifecycleScanState(String bucketKey, long scanStartTime) {
+ this.bucketKey = bucketKey;
+ this.scanStartTime = scanStartTime;
+ }
+
+ private OmLifecycleScanState(Builder builder) {
+ this.bucketKey = builder.bucketKey;
+ this.bucketObjID = builder.bucketObjID;
+ this.lifecycleConfigurationUpdateID = builder.lifecycleConfigurationUpdateID;
+ this.scanStartTime = builder.scanStartTime;
+ this.scanEndTime = builder.scanEndTime;
+ this.lastScannedKey = builder.lastScannedKey;
+ this.lastScannedDir = builder.lastScannedDir;
+ this.lastScannedDirKey = builder.lastScannedDirKey;
+ }
+
+ public String getBucketKey() {
+ return bucketKey;
+ }
+
+ public long getBucketObjID() {
+ return bucketObjID;
+ }
+
+ public long getLifecycleConfigurationUpdateID() {
+ return lifecycleConfigurationUpdateID;
+ }
+
+ public long getScanStartTime() {
+ return scanStartTime;
+ }
+
+ public Long getScanEndTime() {
+ return scanEndTime;
+ }
+
+ public void setScanEndTime(Long scanEndTime) {
+ this.scanEndTime = scanEndTime;
+ }
+
+ public String getLastScannedKey() {
+ return lastScannedKey;
+ }
+
+ public void setLastScannedKey(String lastScannedKey) {
+ this.lastScannedKey = lastScannedKey;
+ }
+
+ public String getLastScannedDir() {
+ return lastScannedDir;
+ }
+
+ public void setLastScannedDir(String dir) {
+ this.lastScannedDir = dir;
+ }
+
+ public String getLastScannedDirKey() {
+ return lastScannedDirKey;
+ }
+
+ public LifecycleScanState getProtobuf() {
+ LifecycleScanState.Builder builder = LifecycleScanState.newBuilder()
+ .setBucketKey(bucketKey)
+ .setBucketObjID(bucketObjID)
+ .setLifecycleConfigurationUpdateID(lifecycleConfigurationUpdateID)
+ .setScanStartTime(scanStartTime);
+
+ if (scanEndTime != null) {
+ builder.setScanEndTime(scanEndTime);
+ }
+ if (lastScannedKey != null) {
+ builder.setLastScannedKey(lastScannedKey);
+ }
+ if (lastScannedDir != null) {
+ builder.setLastScannedDir(lastScannedDir);
+ }
+ if (lastScannedDirKey != null) {
+ builder.setLastScannedDirKey(lastScannedDirKey);
+ }
+ return builder.build();
+ }
+
+ public static OmLifecycleScanState getFromProtobuf(LifecycleScanState proto) {
+ Builder builder = new Builder()
+ .setBucketKey(proto.getBucketKey())
+ .setBucketObjID(proto.getBucketObjID())
+ .setLifecycleConfigurationUpdateID(proto.getLifecycleConfigurationUpdateID())
+ .setScanStartTime(proto.getScanStartTime());
+
+ if (proto.hasScanEndTime()) {
+ builder.setScanEndTime(proto.getScanEndTime());
+ }
+ if (proto.hasLastScannedKey()) {
+ builder.setLastScannedKey(proto.getLastScannedKey());
+ }
+ if (proto.hasLastScannedDir()) {
+ builder.setLastScannedDir(proto.getLastScannedDir());
+ }
+ if (proto.hasLastScannedDirKey()) {
+ builder.setLastScannedDirKey(proto.getLastScannedDirKey());
+ }
+ return builder.build();
+ }
+
+ @Override
+ public String toString() {
+ return "OmLifecycleScanState{" +
+ "bucketKey='" + bucketKey + '\'' +
+ ", bucketObjID=" + bucketObjID +
+ ", lifecycleConfigurationUpdateID=" + lifecycleConfigurationUpdateID +
+ ", scanStartTime=" + scanStartTime +
+ ", scanEndTime=" + scanEndTime +
+ ", lastScannedKey='" + lastScannedKey + '\'' +
+ ", lastScannedDir='" + lastScannedDir + '\'' +
+ ", lastScannedDirKey='" + lastScannedDirKey + '\'' +
+ '}';
+ }
+
+ public Builder toBuilder() {
+ return new Builder()
+ .setBucketKey(bucketKey)
+ .setBucketObjID(bucketObjID)
+ .setLifecycleConfigurationUpdateID(lifecycleConfigurationUpdateID)
+ .setScanStartTime(scanStartTime)
+ .setScanEndTime(scanEndTime)
+ .setLastScannedKey(lastScannedKey)
+ .setLastScannedDir(lastScannedDir)
+ .setLastScannedDirKey(lastScannedDirKey);
+ }
+
+ /**
+ * Builder for OmLifecycleScanState.
+ */
+ public static class Builder {
+ private String bucketKey;
+ private long bucketObjID;
+ private long lifecycleConfigurationUpdateID;
+ private long scanStartTime;
+ private Long scanEndTime;
+ private String lastScannedKey;
+ private String lastScannedDir;
+ private String lastScannedDirKey;
+
+ public Builder setBucketKey(String bucketKey) {
+ this.bucketKey = bucketKey;
+ return this;
+ }
+
+ public long getBucketObjID() {
+ return bucketObjID;
+ }
+
+ public Builder setBucketObjID(long bucketObjID) {
+ this.bucketObjID = bucketObjID;
+ return this;
+ }
+
+ public long getLifecycleConfigurationUpdateID() {
+ return lifecycleConfigurationUpdateID;
+ }
+
+ public Builder setLifecycleConfigurationUpdateID(long lifecycleConfigurationUpdateID) {
+ this.lifecycleConfigurationUpdateID = lifecycleConfigurationUpdateID;
+ return this;
+ }
+
+ public Builder setScanStartTime(long scanStartTime) {
+ this.scanStartTime = scanStartTime;
+ return this;
+ }
+
+ public Builder setScanEndTime(Long scanEndTime) {
+ this.scanEndTime = scanEndTime;
+ return this;
+ }
+
+ public Builder setLastScannedKey(String keyTableKey) {
+ this.lastScannedKey = keyTableKey;
+ return this;
+ }
+
+ public String getLastScannedKey() {
+ return lastScannedKey;
+ }
+
+ public String getLastScannedDir() {
+ return lastScannedDir;
+ }
+
+ public Builder setLastScannedDir(String dirTableKey) {
+ this.lastScannedDir = dirTableKey;
+ return this;
+ }
+
+ public String getLastScannedDirKey() {
+ return lastScannedDirKey;
+ }
+
+ public Builder setLastScannedDirKey(String lastScannedDirKey) {
+ this.lastScannedDirKey = lastScannedDirKey;
+ return this;
+ }
+
+ public OmLifecycleScanState build() {
+ return new OmLifecycleScanState(this);
+ }
+ }
+}
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java
index 15e72e5a8c3..8b39e6b7aac 100644
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java
@@ -104,6 +104,39 @@ public void testCreateInValidOmLCRule() throws OMException {
"Filter and Prefix cannot both be null.");
}
+ @Test
+ public void testCreateFSOLCRule() throws OMException {
+ long currentTime = System.currentTimeMillis();
+ OmLCExpiration exp = new OmLCExpiration.Builder()
+ .setDays(30)
+ .build();
+
+ OmLCRule.Builder r1 = new OmLCRule.Builder()
+ .setId("remove Spark logs after 30 days")
+ .setEnabled(true)
+ .setPrefix("spark/logs")
+ .setAction(exp);
+ assertOMException(() -> r1.build().valid(BucketLayout.FILE_SYSTEM_OPTIMIZED, currentTime),
+ INVALID_REQUEST, "FILE_SYSTEM_OPTIMIZED bucket prefix must end with '/'");
+
+ OmLCRule.Builder r2 = new OmLCRule.Builder()
+ .setEnabled(true)
+ .setPrefix("spark/logs/")
+ .setAction(exp);
+ assertDoesNotThrow(() -> r2.build().valid(BucketLayout.FILE_SYSTEM_OPTIMIZED, currentTime));
+
+ OmLCRule.Builder r3 = new OmLCRule.Builder()
+ .setEnabled(true)
+ .setPrefix("")
+ .setAction(exp);
+ OmLCRule omLCRule = assertDoesNotThrow(r3::build);
+ assertDoesNotThrow(() -> omLCRule.valid(BucketLayout.FILE_SYSTEM_OPTIMIZED, currentTime));
+
+ // Empty id should generate a 48 (default) bit one.
+ assertEquals(OmLCRule.LC_ID_LENGTH, omLCRule.getId().length(),
+ "Expected a " + OmLCRule.LC_ID_LENGTH + " length generated ID");
+ }
+
@Test
public void testMultipleActionsInRule() throws OMException {
long currentTime = System.currentTimeMillis();
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifecycleScanState.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifecycleScanState.java
new file mode 100644
index 00000000000..59244bb06ce
--- /dev/null
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifecycleScanState.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleScanState;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for OmLifecycleScanState.
+ */
+public class TestOmLifecycleScanState {
+
+ @Test
+ public void testBuilderAndProtobufConversion() {
+ OmLifecycleScanState state = new OmLifecycleScanState.Builder()
+ .setBucketKey("/vol1/bucket1")
+ .setScanStartTime(123456789L)
+ .setScanEndTime(123456799L)
+ .setLastScannedKey("key1")
+ .setLastScannedDir("subDir1")
+ .build();
+
+ assertEquals("/vol1/bucket1", state.getBucketKey());
+ assertEquals(123456789L, state.getScanStartTime());
+ assertEquals(123456799L, state.getScanEndTime());
+ assertEquals("key1", state.getLastScannedKey());
+ assertEquals("subDir1", state.getLastScannedDir());
+
+ LifecycleScanState proto = state.getProtobuf();
+ OmLifecycleScanState decodedState = OmLifecycleScanState.getFromProtobuf(proto);
+
+ assertEquals(state.getBucketKey(), decodedState.getBucketKey());
+ assertEquals(state.getScanStartTime(), decodedState.getScanStartTime());
+ assertEquals(state.getScanEndTime(), decodedState.getScanEndTime());
+ assertEquals(state.getLastScannedKey(), decodedState.getLastScannedKey());
+ assertEquals(state.getLastScannedDir(), decodedState.getLastScannedDir());
+ }
+
+ @Test
+ public void testBuilderAndProtobufConversionWithoutOptionals() {
+ OmLifecycleScanState state = new OmLifecycleScanState.Builder()
+ .setBucketKey("/vol1/bucket1")
+ .setScanStartTime(123456789L)
+ .build();
+
+ assertEquals("/vol1/bucket1", state.getBucketKey());
+ assertEquals(123456789L, state.getScanStartTime());
+ assertNull(state.getScanEndTime());
+ assertNull(state.getLastScannedKey());
+ assertNull(state.getLastScannedDir());
+
+ LifecycleScanState proto = state.getProtobuf();
+ OmLifecycleScanState decodedState = OmLifecycleScanState.getFromProtobuf(proto);
+
+ assertEquals(state.getBucketKey(), decodedState.getBucketKey());
+ assertEquals(state.getScanStartTime(), decodedState.getScanStartTime());
+ assertNull(decodedState.getScanEndTime());
+ assertNull(decodedState.getLastScannedKey());
+ assertNull(decodedState.getLastScannedDir());
+ }
+}
diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index d0a110b7263..f2b21a06ad3 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -162,6 +162,7 @@ enum Type {
DeleteLifecycleConfiguration = 146;
GetLifecycleServiceStatus = 147;
SetLifecycleServiceStatus = 148;
+ SaveLifecycleScanState = 149;
}
enum SafeMode {
@@ -319,6 +320,7 @@ message OMRequest {
optional DeleteLifecycleConfigurationRequest deleteLifecycleConfigurationRequest = 147;
optional GetLifecycleServiceStatusRequest getLifecycleServiceStatusRequest = 148;
optional SetLifecycleServiceStatusRequest setLifecycleServiceStatusRequest = 149;
+ optional SaveLifecycleScanStateRequest saveLifecycleScanStateRequest = 150;
}
message OMResponse {
@@ -459,6 +461,7 @@ message OMResponse {
optional DeleteLifecycleConfigurationResponse deleteLifecycleConfigurationResponse = 146;
optional GetLifecycleServiceStatusResponse getLifecycleServiceStatusResponse = 147;
optional SetLifecycleServiceStatusResponse setLifecycleServiceStatusResponse = 148;
+ optional SaveLifecycleScanStateResponse saveLifecycleScanStateResponse = 149;
}
enum Status {
@@ -1414,6 +1417,7 @@ message DeleteKeyRequest {
message DeleteKeysRequest {
optional DeleteKeyArgs deleteKeys = 1;
optional RequestSource sourceType = 2 [default = USER];
+ optional LifecycleScanState scanState = 3;
}
enum RequestSource {
@@ -2624,3 +2628,22 @@ message SetLifecycleServiceStatusRequest {
message SetLifecycleServiceStatusResponse {
}
+
+message LifecycleScanState {
+ optional string bucketKey = 1; // volume/bucket
+ optional uint64 bucketObjID = 2;
+ optional uint64 lifecycleConfigurationUpdateID = 3;
+ optional uint64 scanStartTime = 4;
+ optional uint64 scanEndTime = 5;
+ optional string lastScannedKey = 6;
+ optional string lastScannedDir = 7;
+ optional string lastScannedDirKey = 8;
+ optional string lastScannedMpuKey = 9;
+}
+
+message SaveLifecycleScanStateRequest {
+ optional LifecycleScanState state = 1;
+}
+
+message SaveLifecycleScanStateResponse {
+}
diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
index a664379426d..88e4a03b2cd 100644
--- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
+++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
@@ -49,6 +49,7 @@
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmLifecycleConfiguration;
+import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState;
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartPartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartPartKey;
@@ -499,6 +500,13 @@ String getMultipartKeyFSO(String volume, String bucket, String key, String
Table<String, OmLifecycleConfiguration> getLifecycleConfigurationTable();
+ /**
+ * Gets the LifecycleScanStateTable.
+ *
+ * @return Table
+ */
+ Table<String, OmLifecycleScanState> getLifecycleScanStateTable();
+
/**
* @return list all LifecycleConfigurations.
*/
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index faa91bb67cf..29a7a3943fa 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -109,6 +109,7 @@
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OmLifecycleConfiguration;
+import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState;
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartPartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartPartKey;
@@ -178,6 +179,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager,
private Table<String, TransactionInfo> transactionInfoTable;
private Table<String, String> metaTable;
private Table<String, OmLifecycleConfiguration> lifecycleConfigurationTable;
+ private Table<String, OmLifecycleScanState> lifecycleScanStateTable;
// Tables required for multi-tenancy
private Table<String, OmDBAccessIdInfo> tenantAccessIdTable;
@@ -512,6 +514,7 @@ protected void initializeOmTables(CacheType cacheType,
compactionLogTable = initializer.get(OMDBDefinition.COMPACTION_LOG_TABLE_DEF);
lifecycleConfigurationTable = initializer.get(OMDBDefinition.LIFECYCLE_CONFIGURATION_TABLE_DEF, cacheType);
+ lifecycleScanStateTable = initializer.get(OMDBDefinition.LIFECYCLE_SCAN_STATE_TABLE_DEF, cacheType);
}
/**
@@ -1719,6 +1722,11 @@ public Table<String, OmLifecycleConfiguration> getLifecycleConfigurationTable()
return lifecycleConfigurationTable;
}
+ @Override
+ public Table<String, OmLifecycleScanState> getLifecycleScanStateTable() {
+ return lifecycleScanStateTable;
+ }
+
/**
* @return list all LifecycleConfigurations.
*/
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
index ba583258e83..79284fa0a84 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
@@ -35,6 +35,7 @@
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmLifecycleConfiguration;
+import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState;
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartPartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartPartKey;
@@ -334,6 +335,13 @@ public final class OMDBDefinition extends DBDefinition.WithMap {
StringCodec.get(),
OmLifecycleConfiguration.getCodec());
+ public static final String LIFECYCLE_SCAN_STATE_TABLE =
+ "lifecycleScanStateTable";
+ public static final DBColumnFamilyDefinition<String, OmLifecycleScanState> LIFECYCLE_SCAN_STATE_TABLE_DEF
+ = new DBColumnFamilyDefinition<>(LIFECYCLE_SCAN_STATE_TABLE,
+ StringCodec.get(),
+ OmLifecycleScanState.getCodec());
+
//---------------------------------------------------------------------------
private static final Map<String, DBColumnFamilyDefinition<?, ?>> COLUMN_FAMILIES
= DBColumnFamilyDefinition.newUnmodifiableMap(
@@ -360,7 +368,8 @@ public final class OMDBDefinition extends DBDefinition.WithMap {
TRANSACTION_INFO_TABLE_DEF,
USER_TABLE_DEF,
VOLUME_TABLE_DEF,
- LIFECYCLE_CONFIGURATION_TABLE_DEF);
+ LIFECYCLE_CONFIGURATION_TABLE_DEF,
+ LIFECYCLE_SCAN_STATE_TABLE_DEF);
private static final OMDBDefinition INSTANCE = new OMDBDefinition();
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
index dcab37b348d..04dc20b0afc 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
@@ -65,6 +65,7 @@
import org.apache.hadoop.ozone.om.request.key.acl.prefix.OMPrefixSetAclRequest;
import org.apache.hadoop.ozone.om.request.lifecycle.OMLifecycleConfigurationDeleteRequest;
import org.apache.hadoop.ozone.om.request.lifecycle.OMLifecycleConfigurationSetRequest;
+import org.apache.hadoop.ozone.om.request.lifecycle.OMLifecycleSaveScanStateRequest;
import org.apache.hadoop.ozone.om.request.lifecycle.OMLifecycleSetServiceStatusRequest;
import org.apache.hadoop.ozone.om.request.s3.multipart.S3ExpiredMultipartUploadsAbortRequest;
import org.apache.hadoop.ozone.om.request.s3.security.OMSetSecretRequest;
@@ -351,6 +352,8 @@ public static OMClientRequest createClientRequest(OMRequest omRequest,
return new OMLifecycleConfigurationDeleteRequest(omRequest);
case SetLifecycleServiceStatus:
return new OMLifecycleSetServiceStatusRequest(omRequest);
+ case SaveLifecycleScanState:
+ return new OMLifecycleSaveScanStateRequest(omRequest);
default:
throw new OMException("Unrecognized write command type request "
+ cmdType, OMException.ResultCodes.INVALID_REQUEST);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java
index 7ee4400f52c..528badf912a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java
@@ -40,6 +40,7 @@
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
@@ -57,6 +58,7 @@
import org.apache.hadoop.ozone.om.helpers.ErrorInfo;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator;
@@ -75,6 +77,7 @@
import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -91,6 +94,25 @@ public OMKeysDeleteRequest(OMRequest omRequest, BucketLayout bucketLayout) {
super(omRequest, bucketLayout);
}
+ @Override
+ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
+ DeleteKeysRequest deleteKeysRequest = super.preExecute(ozoneManager).getDeleteKeysRequest();
+ Objects.requireNonNull(deleteKeysRequest, "deleteKeysRequest == null");
+
+ if (deleteKeysRequest.getSourceType() == RequestSource.LIFECYCLE && deleteKeysRequest.hasScanState()) {
+ if (ozoneManager.getAclsEnabled()) {
+ UserGroupInformation ugi = createUGIForApi();
+ if (!ozoneManager.isAdmin(ugi)) {
+ throw new OMException("Access denied for user " + ugi + ". "
+ + "Superuser privilege is required to save Lifecycle Service task state.",
+ OMException.ResultCodes.ACCESS_DENIED);
+ }
+ }
+ }
+
+ return getOmRequest();
+ }
+
@Override @SuppressWarnings("methodlength")
public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, ExecutionContext context) {
final long trxnLogIndex = context.getIndex();
@@ -227,10 +249,19 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
quotaReleasedEmptyKeys.getValue(), true);
omBucketInfo.decrUsedNamespace(quotaReleasedEmptyKeys.getValue(), false);
+ OmLifecycleScanState state = null;
+ if (sourceType == RequestSource.LIFECYCLE && deleteKeyRequest.hasScanState()) {
+ state = OmLifecycleScanState.getFromProtobuf(deleteKeyRequest.getScanState());
+ // Update cache
+ ozoneManager.getMetadataManager().getLifecycleScanStateTable()
+ .addCacheEntry(new CacheKey<>(state.getBucketKey()),
+ CacheValue.get(trxnLogIndex, state));
+ }
+
final long volumeId = omMetadataManager.getVolumeId(volumeName);
omClientResponse =
getOmClientResponse(ozoneManager, omKeyInfoList, dirList, omResponse,
- unDeletedKeys, keyToError, deleteStatus, omBucketInfo, volumeId, openKeyInfoMap);
+ unDeletedKeys, keyToError, deleteStatus, omBucketInfo, volumeId, openKeyInfoMap, state);
result = Result.SUCCESS;
long endNanosDeleteKeySuccessLatencyNs = Time.monotonicNowNanos();
@@ -335,7 +366,8 @@ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager,
OMResponse.Builder omResponse,
OzoneManagerProtocolProtos.DeleteKeyArgs.Builder unDeletedKeys,
Map<String, ErrorInfo> keyToErrors,
- boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId, Map<String, OmKeyInfo> openKeyInfoMap) {
+ boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId, Map<String, OmKeyInfo> openKeyInfoMap,
+ OmLifecycleScanState scanState) {
OMClientResponse omClientResponse;
List<OzoneManagerProtocolProtos.DeleteKeyError> deleteKeyErrors = new ArrayList<>();
for (Map.Entry<String, ErrorInfo> key : keyToErrors.entrySet()) {
@@ -348,7 +380,7 @@ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager,
.setUnDeletedKeys(unDeletedKeys).addAllErrors(deleteKeyErrors))
.setStatus(deleteStatus ? OK : PARTIAL_DELETE).setSuccess(deleteStatus)
.build(), omKeyInfoList,
- omBucketInfo.copyObject(), openKeyInfoMap);
+ omBucketInfo.copyObject(), openKeyInfoMap, scanState);
return omClientResponse;
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java
index a501739d0c3..51de2dd0753 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java
@@ -37,6 +37,7 @@
import org.apache.hadoop.ozone.om.helpers.ErrorInfo;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
@@ -165,7 +166,8 @@ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager,
OzoneManagerProtocolProtos.OMResponse.Builder omResponse,
OzoneManagerProtocolProtos.DeleteKeyArgs.Builder unDeletedKeys,
Map<String, ErrorInfo> keyToErrors,
- boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId, Map<String, OmKeyInfo> openKeyInfoMap) {
+ boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId, Map<String,
+ OmKeyInfo> openKeyInfoMap, OmLifecycleScanState state) {
OMClientResponse omClientResponse;
List<OzoneManagerProtocolProtos.DeleteKeyError> deleteKeyErrors = new ArrayList<>();
for (Map.Entry<String, ErrorInfo> key : keyToErrors.entrySet()) {
@@ -179,7 +181,7 @@ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager,
.setStatus(deleteStatus).setUnDeletedKeys(unDeletedKeys).addAllErrors(deleteKeyErrors))
.setStatus(deleteStatus ? OK : PARTIAL_DELETE).setSuccess(deleteStatus)
.build(), omKeyInfoList, dirList,
- omBucketInfo.copyObject(), volumeId, openKeyInfoMap);
+ omBucketInfo.copyObject(), volumeId, openKeyInfoMap, state);
return omClientResponse;
}
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleSaveScanStateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleSaveScanStateRequest.java
new file mode 100644
index 00000000000..94ef71f0b01
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleSaveScanStateRequest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.request.lifecycle;
+
+import java.io.IOException;
+import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
+import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext;
+import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState;
+import org.apache.hadoop.ozone.om.request.OMClientRequest;
+import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
+import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.om.response.lifecycle.OMLifecycleSaveScanStateResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SaveLifecycleScanStateRequest;
+import org.apache.hadoop.security.UserGroupInformation;
+
+/**
+ * Handles SaveLifecycleScanState request.
+ */
+public class OMLifecycleSaveScanStateRequest extends OMClientRequest {
+
+ public OMLifecycleSaveScanStateRequest(OMRequest omRequest) {
+ super(omRequest);
+ }
+
+ @Override
+ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException {
+ OMRequest omRequest = super.preExecute(ozoneManager);
+ if (ozoneManager.getAclsEnabled()) {
+ UserGroupInformation ugi = createUGIForApi();
+ if (!ozoneManager.isAdmin(ugi)) {
+ throw new OMException("Access denied for user " + ugi + ". "
+ + "Superuser privilege is required to save Lifecycle Service task state.",
+ OMException.ResultCodes.ACCESS_DENIED);
+ }
+ }
+ return omRequest;
+ }
+
+ @Override
+ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, ExecutionContext context) {
+ SaveLifecycleScanStateRequest request = getOmRequest().getSaveLifecycleScanStateRequest();
+ OmLifecycleScanState state = OmLifecycleScanState.getFromProtobuf(request.getState());
+
+ OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
+ getOmRequest());
+
+ // Update cache
+ ozoneManager.getMetadataManager().getLifecycleScanStateTable()
+ .addCacheEntry(new CacheKey<>(state.getBucketKey()),
+ CacheValue.get(context.getIndex(), state));
+
+ return new OMLifecycleSaveScanStateResponse(omResponse.build(), state);
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java
index 3cb1220b83c..fd089005fcd 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java
@@ -20,6 +20,7 @@
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.BUCKET_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DELETED_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.LIFECYCLE_SCAN_STATE_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.OPEN_KEY_TABLE;
import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK;
import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.PARTIAL_DELETE;
@@ -35,26 +36,29 @@
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState;
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
/**
* Response for DeleteKey request.
*/
-@CleanupTableInfo(cleanupTables = {KEY_TABLE, OPEN_KEY_TABLE, DELETED_TABLE, BUCKET_TABLE})
+@CleanupTableInfo(cleanupTables = {KEY_TABLE, OPEN_KEY_TABLE, DELETED_TABLE, BUCKET_TABLE, LIFECYCLE_SCAN_STATE_TABLE})
public class OMKeysDeleteResponse extends AbstractOMKeyDeleteResponse {
private List<OmKeyInfo> omKeyInfoList;
private OmBucketInfo omBucketInfo;
private Map<String, OmKeyInfo> openKeyInfoMap = new HashMap<>();
+ private OmLifecycleScanState scanState;
public OMKeysDeleteResponse(@Nonnull OMResponse omResponse,
@Nonnull List<OmKeyInfo> keyDeleteList,
@Nonnull OmBucketInfo omBucketInfo,
- @Nonnull Map<String, OmKeyInfo> openKeyInfoMap) {
+ @Nonnull Map<String, OmKeyInfo> openKeyInfoMap, OmLifecycleScanState scanState) {
super(omResponse);
this.omKeyInfoList = keyDeleteList;
this.omBucketInfo = omBucketInfo;
this.openKeyInfoMap = openKeyInfoMap;
+ this.scanState = scanState;
}
/**
@@ -107,6 +111,11 @@ public void addToDBBatch(OMMetadataManager omMetadataManager,
batchOperation, entry.getKey(), entry.getValue());
}
}
+
+ if (scanState != null) {
+ omMetadataManager.getLifecycleScanStateTable().putWithBatch(
+ batchOperation, scanState.getBucketKey(), scanState);
+ }
}
public List<OmKeyInfo> getOmKeyInfoList() {
@@ -120,4 +129,8 @@ public OmBucketInfo getOmBucketInfo() {
protected Map<String, OmKeyInfo> getOpenKeyInfoMap() {
return openKeyInfoMap;
}
+
+ public OmLifecycleScanState getScanState() {
+ return scanState;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java
index 0b283509354..7bdcfa00b5b 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java
@@ -22,6 +22,7 @@
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DELETED_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DIRECTORY_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.LIFECYCLE_SCAN_STATE_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.OPEN_FILE_TABLE;
import jakarta.annotation.Nonnull;
@@ -34,6 +35,7 @@
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState;
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
@@ -41,7 +43,7 @@
* Response for DeleteKeys request.
*/
@CleanupTableInfo(cleanupTables = { FILE_TABLE, OPEN_FILE_TABLE, DIRECTORY_TABLE,
- DELETED_DIR_TABLE, DELETED_TABLE, BUCKET_TABLE })
+ DELETED_DIR_TABLE, DELETED_TABLE, BUCKET_TABLE, LIFECYCLE_SCAN_STATE_TABLE})
public class OMKeysDeleteResponseWithFSO extends OMKeysDeleteResponse {
private List<OmKeyInfo> dirsList;
@@ -52,8 +54,9 @@ public OMKeysDeleteResponseWithFSO(
@Nonnull List<OmKeyInfo> keyDeleteList,
@Nonnull List<OmKeyInfo> dirDeleteList,
@Nonnull OmBucketInfo omBucketInfo, @Nonnull long volId,
- @Nonnull Map<String, OmKeyInfo> openKeyInfoMap) {
- super(omResponse, keyDeleteList, omBucketInfo, openKeyInfoMap);
+ @Nonnull Map<String, OmKeyInfo> openKeyInfoMap,
+ OmLifecycleScanState scanState) {
+ super(omResponse, keyDeleteList, omBucketInfo, openKeyInfoMap, scanState);
this.dirsList = dirDeleteList;
this.volumeId = volId;
}
@@ -101,6 +104,12 @@ public void addToDBBatch(OMMetadataManager omMetadataManager,
batchOperation, entry.getKey(), entry.getValue());
}
}
+
+ OmLifecycleScanState scanState = getScanState();
+ if (scanState != null) {
+ omMetadataManager.getLifecycleScanStateTable().putWithBatch(
+ batchOperation, scanState.getBucketKey(), scanState);
+ }
}
@Override
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleSaveScanStateResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleSaveScanStateResponse.java
new file mode 100644
index 00000000000..0f94293269b
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleSaveScanStateResponse.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.response.lifecycle;
+
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.LIFECYCLE_SCAN_STATE_TABLE;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK;
+
+import java.io.IOException;
+import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState;
+import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
+import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+
+/**
+ * Response for SaveLifecycleScanState request.
+ */
+@CleanupTableInfo(cleanupTables = {LIFECYCLE_SCAN_STATE_TABLE})
+public class OMLifecycleSaveScanStateResponse extends OMClientResponse {
+
+ private OmLifecycleScanState state;
+
+ public OMLifecycleSaveScanStateResponse(OMResponse omResponse, OmLifecycleScanState state) {
+ super(omResponse);
+ this.state = state;
+ }
+
+ @Override
+ public void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException {
+ if (getOMResponse().getStatus() == OK) {
+ omMetadataManager.getLifecycleScanStateTable().putWithBatch(
+ batchOperation, state.getBucketKey(), state);
+ }
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
index dd37430dccf..a3df126fdb0 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
@@ -30,6 +30,10 @@
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_MPU_ABORT_LIMIT_PER_TASK;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_MPU_ABORT_LIMIT_PER_TASK_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT;
import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE;
import com.google.common.annotations.VisibleForTesting;
@@ -42,7 +46,8 @@
import java.time.Instant;
import java.util.ArrayDeque;
import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.Deque;
import java.util.HashSet;
import java.util.Iterator;
@@ -83,6 +88,7 @@
import org.apache.hadoop.ozone.om.helpers.OmLCFilter;
import org.apache.hadoop.ozone.om.helpers.OmLCRule;
import org.apache.hadoop.ozone.om.helpers.OmLifecycleConfiguration;
+import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState;
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUpload;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
@@ -99,6 +105,7 @@
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeyRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RequestSource;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SaveLifecycleScanStateRequest;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Time;
import org.apache.ratis.protocol.ClientId;
@@ -125,10 +132,14 @@ public class KeyLifecycleService extends BackgroundService {
private final ConcurrentHashMap<String, LifecycleActionTask> inFlight;
private OMMetadataManager omMetadataManager;
private int ratisByteLimit;
+ private long stateSaveIntervalMs;
+ private long maxKeysProcessedPerState;
private ClientId clientId = ClientId.randomId();
private AtomicLong callId = new AtomicLong(0);
private OzoneTrash ozoneTrash;
private static List<FaultInjector> injectors;
+ private static boolean test = false;
+ private static List<RuleListWithDirectoryList> consolidatedRuleList;
public KeyLifecycleService(OzoneManager ozoneManager,
KeyManager manager, long serviceInterval,
@@ -154,6 +165,23 @@ public KeyLifecycleService(OzoneManager ozoneManager,
OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT));
this.moveToTrashEnabled = new AtomicBoolean(conf.getBoolean(OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED,
OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED_DEFAULT));
+ this.stateSaveIntervalMs = conf.getLong(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS,
+ OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT);
+ if (!test && stateSaveIntervalMs <= 0) {
+ LOG.warn("Illegal value {} for Property {}. Set {} to {}", stateSaveIntervalMs,
+ OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS, OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS,
+ OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT);
+ stateSaveIntervalMs = OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT;
+ }
+ this.maxKeysProcessedPerState = conf.getLong(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED,
+ OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT);
+ if (!test && maxKeysProcessedPerState <= 0) {
+ LOG.warn("Illegal value {} for Property {}. Set {} to {}", maxKeysProcessedPerState,
+ OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED, OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED,
+ OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT);
+ maxKeysProcessedPerState = OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT;
+ }
+ LOG.info("stateSaveIntervalMs = {}, maxKeysProcessedPerState = {}", stateSaveIntervalMs, maxKeysProcessedPerState);
this.inFlight = new ConcurrentHashMap();
this.omMetadataManager = ozoneManager.getMetadataManager();
int limit = (int) conf.getStorageSize(
@@ -278,6 +306,20 @@ public final class LifecycleActionTask implements BackgroundTask {
private long numDirRenamed = 0;
private long numMultipartUploadIterated = 0;
private long numMultipartUploadAborted = 0;
+ private String lastScannedKey;
+ private String lastScannedDir;
+ private String lastScannedDirKey;
+
+ private long lastStateSaveTime = Time.monotonicNow();
+ private long lastStateSaveKeyCount = 0;
+
+ private boolean shouldSaveState() {
+ if ((Time.monotonicNow() - lastStateSaveTime) > stateSaveIntervalMs ||
+ (numKeyIterated - lastStateSaveKeyCount) >= maxKeysProcessedPerState) {
+ return true;
+ }
+ return false;
+ }
public LifecycleActionTask(OmLifecycleConfiguration lcConfig) {
this.policy = lcConfig;
@@ -296,6 +338,7 @@ public BackgroundTaskResult call() {
if (shouldRun()) {
LOG.info("Running LifecycleActionTask {}", bucketKey);
taskStartTime = Time.monotonicNow();
+ lastStateSaveTime = taskStartTime;
OmBucketInfo bucket;
try {
if (getInjector(0) != null) {
@@ -319,76 +362,108 @@ public BackgroundTaskResult call() {
return result;
}
- List<OmLCRule> originRuleList = policy.getRules();
- // remove disabled rules
- List<OmLCRule> ruleList = originRuleList.stream().filter(r -> r.isEnabled()).collect(Collectors.toList());
-
- List<OmLCRule> expirationRules = ruleList.stream()
- .filter(r -> r.getExpiration() != null)
- .collect(Collectors.toList());
- List<OmLCRule> mpuRules = ruleList.stream()
- .filter(r -> r.getAbortIncompleteMultipartUpload() != null)
- .collect(Collectors.toList());
-
- if (!expirationRules.isEmpty()) {
- LimitedExpiredObjectList expiredKeyList = new LimitedExpiredObjectList(listMaxSize);
- LimitedExpiredObjectList expiredDirList = new LimitedExpiredObjectList(listMaxSize);
- Table<String, OmKeyInfo> keyTable = omMetadataManager.getKeyTable(bucket.getBucketLayout());
- /**
- * Filter treatment.
- * "" - all objects
- * "/" - if it's OBS/Legacy, means keys starting with "/"; If it's FSO, not supported
- * "/key" - if it's OBS/Legacy, means keys starting with "/key", "/" is literally "/";
- * If it's FSO, means keys or dirs starting with "key", "/" will be treated as separator mark.
- * "key" - if it's OBS/Legacy, means keys starting with "key";
- * if it's FSO, means keys for dirs starting with "key" too.
- * "dir/" - if it's OBS/Legacy, means keys starting with "dir/";
- * - if it's FSO, means keys/dirs under directory "dir", doesn't include directory "dir" itself.
- * - For FSO bucket, as directory ModificationTime will not be updated when any of its child key/subdir
- * changes, so remember to add the tailing slash "/" when configure prefix, otherwise the whole
- * directory will be expired and deleted once its ModificationTime meats the condition.
- */
- if (bucket.getBucketLayout() == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
- OmVolumeArgs volume;
- try {
- volume = omMetadataManager.getVolumeTable().get(omMetadataManager.getVolumeKey(bucket.getVolumeName()));
- if (volume == null) {
- LOG.warn("Volume {} cannot be found, might be deleted during this task's execution",
- bucket.getVolumeName());
+ OmLifecycleScanState.Builder scanStateBuilder = null;
+ try {
+ OmLifecycleScanState scanState = omMetadataManager.getLifecycleScanStateTable().get(bucketKey);
+ if (scanState == null || (scanState.getBucketObjID() != bucket.getObjectID() ||
+ scanState.getLifecycleConfigurationUpdateID() != policy.getUpdateID() ||
+ scanState.getScanEndTime() != null)) {
+ scanStateBuilder = new OmLifecycleScanState.Builder();
+ scanStateBuilder.setBucketKey(bucketKey);
+ scanStateBuilder.setScanStartTime(System.currentTimeMillis());
+ scanStateBuilder.setBucketObjID(bucket.getObjectID());
+ scanStateBuilder.setLifecycleConfigurationUpdateID(policy.getUpdateID());
+ LOG.info("Create/Recreate OmLifecycleScanState for {} bucket {} bucketID {} " +
+ "lifecycleConfigurationUpdateID {}", bucket.getBucketLayout(), bucketKey, bucket.getObjectID(),
+ policy.getUpdateID());
+ } else {
+ scanStateBuilder = scanState.toBuilder();
+ LOG.info("Resume OmLifecycleScanState {}", scanState);
+ }
+ } catch (Exception e) {
+ LOG.warn("Failed to get scan state for bucket {}", bucketKey, e);
+ }
+
+ try {
+ List<OmLCRule> originRuleList = policy.getRules();
+ // remove disabled rules
+ List<OmLCRule> ruleList = originRuleList.stream().filter(
+ r -> r.isEnabled()).collect(Collectors.toList());
+
+ List<OmLCRule> expirationRules = ruleList.stream()
+ .filter(r -> r.getExpiration() != null)
+ .collect(Collectors.toList());
+ List<OmLCRule> mpuRules = ruleList.stream()
+ .filter(r -> r.getAbortIncompleteMultipartUpload() != null)
+ .collect(Collectors.toList());
+
+ if (!expirationRules.isEmpty()) {
+ LimitedExpiredObjectList expiredKeyList = new LimitedExpiredObjectList(listMaxSize);
+ LimitedExpiredObjectList expiredDirList = new LimitedExpiredObjectList(listMaxSize);
+ Table<String, OmKeyInfo> keyTable = omMetadataManager.getKeyTable(bucket.getBucketLayout());
+ /**
+ * Filter treatment.
+ * "" - all objects
+ * "/" - if it's OBS/Legacy, means keys starting with "/"; If it's FSO, not supported
+ * "/key" - if it's OBS/Legacy, means keys starting with "/key", "/" is literally "/";
+ * If it's FSO, means keys or dirs starting with "key", "/" will be treated as separator mark.
+ * "key" - if it's OBS/Legacy, means keys starting with "key";
+ * if it's FSO, means keys for dirs starting with "key" too.
+ * "dir/" - if it's OBS/Legacy, means keys starting with "dir/";
+ * - if it's FSO, means keys/dirs under directory "dir", doesn't include directory "dir" itself.
+ * - For FSO bucket, as directory ModificationTime will not be updated when any of its child
+ * key/subdir changes, so remember to add the tailing slash "/" when configure prefix, otherwise
+ * the whole directory will be expired and deleted once its ModificationTime meats the condition.
+ */
+ if (bucket.getBucketLayout() == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
+ OmVolumeArgs volume;
+ try {
+ volume = omMetadataManager.getVolumeTable().get(omMetadataManager.getVolumeKey(bucket.getVolumeName()));
+ if (volume == null) {
+ LOG.warn("Volume {} cannot be found, might be deleted during this task's execution",
+ bucket.getVolumeName());
+ onFailure(bucketKey);
+ return result;
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to get volume {}", bucket.getVolumeName(), e);
onFailure(bucketKey);
return result;
}
- } catch (IOException e) {
- LOG.warn("Failed to get volume {}", bucket.getVolumeName(), e);
- onFailure(bucketKey);
- return result;
+ evaluateFSOBucket(volume, bucket, bucketKey, keyTable, expirationRules, expiredKeyList,
+ expiredDirList, scanStateBuilder);
+ } else {
+ // use bucket name as key iterator prefix
+ evaluateBucket(bucket, keyTable, expirationRules, expiredKeyList, scanStateBuilder);
}
- evaluateFSOBucket(volume, bucket, bucketKey, keyTable, expirationRules, expiredKeyList, expiredDirList);
- } else {
- // use bucket name as key iterator prefix
- evaluateBucket(bucket, keyTable, expirationRules, expiredKeyList);
- }
- if (expiredKeyList.isEmpty() && expiredDirList.isEmpty()) {
- LOG.info("No expired keys/dirs found/remained for bucket {}", bucketKey);
- } else {
- LOG.info("{} expired keys and {} expired dirs found and remained for bucket {}",
- expiredKeyList.size(), expiredDirList.size(), bucketKey);
-
- // If trash is enabled, move files to trash, instead of send delete requests.
- // OBS bucket doesn't support trash.
- if (bucket.getBucketLayout() == OBJECT_STORE) {
- sendDeleteKeysRequestAndClearList(bucket.getVolumeName(), bucket.getBucketName(), expiredKeyList, false);
+ if (expiredKeyList.isEmpty() && expiredDirList.isEmpty()) {
+ LOG.info("No expired keys/dirs found/remained for bucket {}", bucketKey);
+ sendSaveScanStateRequest(scanStateBuilder, true);
} else {
- // handle keys first, then directories
- handleAndClearFullList(bucket, expiredKeyList, false);
- handleAndClearFullList(bucket, expiredDirList, true);
+ LOG.info("{} expired keys and {} expired dirs found and remained for bucket {}",
+ expiredKeyList.size(), expiredDirList.size(), bucketKey);
+
+ // If trash is enabled, move files to trash, instead of send delete requests.
+ // OBS bucket doesn't support trash.
+ if (bucket.getBucketLayout() == OBJECT_STORE) {
+ sendDeleteKeysRequestAndClearList(bucket.getVolumeName(), bucket.getBucketName(), expiredKeyList,
+ false, scanStateBuilder, true);
+ } else {
+ // handle keys first, then directories
+ handleAndClearFullList(bucket, expiredKeyList, false, scanStateBuilder, true);
+ handleAndClearFullList(bucket, expiredDirList, true, scanStateBuilder, true);
+ }
}
}
- }
- if (!mpuRules.isEmpty()) {
- processMultipartUploads(bucket, mpuRules);
+ if (!mpuRules.isEmpty()) {
+ processMultipartUploads(bucket, mpuRules);
+ }
+ } catch (Throwable e) {
+ LOG.error("Failed to evaluate lifecycle configuration for bucket {}", bucketKey, e);
+ onFailure(bucketKey);
+ return result;
}
onSuccess(bucketKey);
@@ -401,108 +476,194 @@ public BackgroundTaskResult call() {
@SuppressWarnings("checkstyle:parameternumber")
private void evaluateFSOBucket(OmVolumeArgs volume, OmBucketInfo bucket, String bucketKey,
Table<String, OmKeyInfo> keyTable, List<OmLCRule> ruleList,
- LimitedExpiredObjectList expiredKeyList, LimitedExpiredObjectList expiredDirList) {
+ LimitedExpiredObjectList expiredKeyList, LimitedExpiredObjectList expiredDirList,
+ OmLifecycleScanState.Builder scanStateBuilder) {
List<OmLCRule> prefixRuleList =
ruleList.stream().filter(r -> r.isPrefixEnable()).collect(Collectors.toList());
// r.isPrefixEnable() == false means empty filter
List<OmLCRule> noPrefixRuleList =
ruleList.stream().filter(r -> !r.isPrefixEnable()).collect(Collectors.toList());
- for (OmLCRule rule : prefixRuleList) {
- // find KeyInfo of each directory for prefix
- List<OmDirectoryInfo> dirList;
+ if (!noPrefixRuleList.isEmpty()) {
+ // evaluate all rules against each key
+ prefixRuleList.addAll(noPrefixRuleList);
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "", null, "",
+ prefixRuleList, expiredKeyList, expiredDirList, scanStateBuilder);
+ return;
+ }
+
+ List<RuleListWithDirectoryList> unionPrefixRuleList =
+ getRuleUnion(volume.getObjectID(), bucket, prefixRuleList, bucketKey);
+
+ if (unionPrefixRuleList != null) {
+ if (unionPrefixRuleList.isEmpty()) {
+ // fallback to evaluate the whole bucket
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "", null, "",
+ prefixRuleList, expiredKeyList, expiredDirList, scanStateBuilder);
+ } else {
+ for (RuleListWithDirectoryList ruleWithDirList : unionPrefixRuleList) {
+ List<OmLCRule> rules = ruleWithDirList.getRuleList();
+ DirectoryList dir = ruleWithDirList.getDirList();
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, dir.getLastSubDirPath(),
+ dir.getLastSubDir(), dir.getLastSubDirKey(),
+ rules, expiredKeyList, expiredDirList, scanStateBuilder);
+ }
+ }
+ }
+ }
+
+ /**
+ * Finds the directory union list from a list of prefixes and sorts them
+ * according to the FSO depth-first iteration order.
+ */
+ private List<RuleListWithDirectoryList> getRuleUnion(long volumeId, OmBucketInfo bucket,
+ List<OmLCRule> rules, String bucketKey) {
+
+ if (rules.isEmpty() || rules.stream().anyMatch(
+ r -> r.getEffectivePrefix() == null || r.getEffectivePrefix().isEmpty())) {
+ // The union of anything with the root is just the root itself.
+ return new ArrayList();
+ }
+
+ List<RuleListWithDirectoryList> effectiveRuleList = new ArrayList<>();
+ for (OmLCRule rule : rules) {
+ String prefix = rule.getEffectivePrefix();
+ // Resolve each prefix to actual FSO directories in the DB
try {
- dirList = getDirList(volume, bucket, rule.getEffectivePrefix(), bucketKey);
+ if (!prefix.endsWith(OzoneConsts.OM_KEY_PREFIX)) {
+ // FSO bucket doesn't allow prefix without tailing '/'
+ // Prefix ends with a slash, it explicitly refers to a directory (e.g. "log/")
+ LOG.warn("Skip rule {} since FILE_SYSTEM_OPTIMIZED bucket prefix must end with '/'", rule);
+ continue;
+ }
+
+ // Normalize by removing the trailing slash for uniform comparison
+ String normalizedPrefix = prefix.substring(0, prefix.length() - 1);
+ DirectoryList dirList = getDirList(volumeId, bucket, normalizedPrefix, bucketKey);
+ // If the prefix is log/, and "log" dir really exists, then the matched dir is "log".
+ // Otherwise, this rule doesn't match any dir/file in this FSO bucket, this rule can be skipped.
+ if (!dirList.isEmpty() && dirList.isAllResolvedPrefix()) {
+ RuleListWithDirectoryList ruleListWithDirectoryList = new RuleListWithDirectoryList(
+ Collections.singletonList(rule), dirList, prefix);
+ effectiveRuleList.add(ruleListWithDirectoryList);
+ }
} catch (IOException e) {
- LOG.warn("Skip rule {} as its prefix doesn't have all directory exist", rule);
- // skip this rule if some directory doesn't exist for this rule's prefix
+ // Directory doesn't exist or IO error, skip this rule
+ LOG.warn("Skip to evaluate rule {} due to failed to resolve prefix {} for bucket {}",
+ rule, prefix, bucketKey, e);
+ }
+ }
+
+ if (effectiveRuleList.isEmpty()) {
+ // there is no valid rule found, either prefix doesn't end with "/",
+ // or any directory along the prefix cannot be found.
+ LOG.warn("Prefix of all rules of bucket {} cannot be resolved to an existing directory. ", bucketKey);
+ return null;
+ }
+
+ if (effectiveRuleList.size() == 1) {
+ return effectiveRuleList;
+ }
+
+ // Find if one rule's prefix is the sub string of another rule's prefix.
+ // e.g.
+ // dir1/dir2/, dir1/dir2/dir3/, dir1/ -> dir1/
+ // dir1/dir2/, dir1/dir3/, dir1/dir4/ -> dir1/dir2/, dir1/dir3/, dir1/dir4dir1/
+ // dir1/dir2/dir3/, dir1/dir2/, dir2/ -> dir1/dir2/, dir2/
+ // dir1/dir2/, dir1/dir3/, dir1/ -> dir1/
+ List<RuleListWithDirectoryList> consolidatedRules = new ArrayList<>();
+ Set<OmLCRule> skipEvaluatedRuleList = new HashSet<>();
+ for (int i = 0; i < effectiveRuleList.size(); i++) {
+ OmLCRule rule = effectiveRuleList.get(i).getRuleList().get(0);
+ if (skipEvaluatedRuleList.contains(rule)) {
continue;
}
- StringBuffer lastDirPath = new StringBuffer();
- OmDirectoryInfo lastDir = null;
- if (!dirList.isEmpty()) {
- lastDir = dirList.get(dirList.size() - 1);
- for (int i = 0; i < dirList.size(); i++) {
- lastDirPath.append(dirList.get(i).getName());
- if (i != dirList.size() - 1) {
- lastDirPath.append(OM_KEY_PREFIX);
- }
- }
- if (lastDirPath.toString().startsWith(TRASH_PREFIX)) {
- LOG.info("Skip evaluate trash directory {}", lastDirPath);
- } else {
- evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, lastDirPath.toString(), lastDir,
- Arrays.asList(rule), expiredKeyList, expiredDirList);
+
+ RuleListWithDirectoryList consolidatedCandidate = new RuleListWithDirectoryList();
+ String consolidatedPrefix = effectiveRuleList.get(i).getConsolidatedPrefix();
+ String finalRuleIndexID = rule.getId();
+ DirectoryList finalDirList = effectiveRuleList.get(i).getDirList();
+ for (int j = i + 1; j < effectiveRuleList.size(); j++) {
+ OmLCRule otherRule = effectiveRuleList.get(j).getRuleList().get(0);
+ if (skipEvaluatedRuleList.contains(otherRule)) {
+ continue;
}
- if (!rule.getEffectivePrefix().endsWith(OM_KEY_PREFIX)) {
- // if the prefix doesn't end with "/", then also search and evaluate the directory itself
- // for example, "dir1/dir2" matches both directory "dir1/dir2" and "dir1/dir22"
- // or "dir1" matches both directory "dir1" and "dir11"
- long objID;
- String objPrefix;
- String objPath;
- if (dirList.size() > 1) {
- OmDirectoryInfo secondLastDir = dirList.get(dirList.size() - 2);
- objID = secondLastDir.getObjectID();
- objPrefix = OM_KEY_PREFIX + volume.getObjectID() + OM_KEY_PREFIX + bucket.getObjectID() +
- OM_KEY_PREFIX + secondLastDir.getObjectID();
- StringBuffer secondLastDirPath = new StringBuffer();
- for (int i = 0; i < dirList.size() - 1; i++) {
- secondLastDirPath.append(dirList.get(i).getName());
- if (i != dirList.size() - 2) {
- secondLastDirPath.append(OM_KEY_PREFIX);
- }
- }
- objPath = secondLastDirPath.toString();
- } else {
- objID = bucket.getObjectID();
- objPrefix = OM_KEY_PREFIX + volume.getObjectID() + OM_KEY_PREFIX + bucket.getObjectID() +
- OM_KEY_PREFIX + bucket.getObjectID();
- objPath = "";
- }
- try {
- SubDirectorySummary subDirSummary = getSubDirectory(objID, objPrefix, omMetadataManager);
- for (OmDirectoryInfo subDir : subDirSummary.getSubDirList()) {
- String subDirPath = objPath.isEmpty() ? subDir.getName() : objPath + OM_KEY_PREFIX + subDir.getName();
- if (!subDir.getName().equals(TRASH_PREFIX) && subDirPath.startsWith(rule.getEffectivePrefix()) &&
- (lastDir == null || subDir.getObjectID() != lastDir.getObjectID())) {
- evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, subDirPath, subDir,
- Arrays.asList(rule), expiredKeyList, expiredDirList);
- }
- }
- } catch (IOException e) {
- // log failure and continue the process
- LOG.warn("Failed to get sub directories of {} under {}/{}", objPrefix,
- bucket.getVolumeName(), bucket.getBucketName(), e);
- return;
- }
+ DirectoryList otherDirList = effectiveRuleList.get(j).getDirList();
+ String otherPrefix = otherRule.getEffectivePrefix();
+ if (otherPrefix.startsWith(consolidatedPrefix)) {
+ LOG.info("Rule {}'s prefix {} is sub string of rule {}'s prefix {}. " +
+ " Consolidate {} into {}.", otherRule.getId(), otherPrefix, finalRuleIndexID,
+ consolidatedPrefix, otherRule.getId(), finalRuleIndexID);
+ consolidatedCandidate.addRule(otherRule);
+ skipEvaluatedRuleList.add(otherRule);
+ } else if (consolidatedPrefix.startsWith(otherPrefix)) {
+ LOG.info("Rule {}'s prefix {} is sub string of rule {}'s prefix {}. Consolidate {} int {}. ",
+ consolidatedPrefix, consolidatedPrefix, otherRule.getId(), otherPrefix, consolidatedPrefix,
+ otherRule.getId());
+ consolidatedPrefix = otherPrefix;
+ finalRuleIndexID = otherRule.getId();
+ finalDirList = otherDirList;
+ consolidatedCandidate.addRule(otherRule);
+ skipEvaluatedRuleList.add(otherRule);
}
- } else {
- evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "", null,
- Arrays.asList(rule), expiredKeyList, expiredDirList);
}
+
+ consolidatedCandidate.addRule(rule);
+ consolidatedCandidate.setDirList(finalDirList);
+ consolidatedCandidate.setConsolidatedPrefix(consolidatedPrefix);
+ consolidatedRules.add(consolidatedCandidate);
}
- if (!noPrefixRuleList.isEmpty()) {
- evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "", null,
- noPrefixRuleList, expiredKeyList, expiredDirList);
+ // Sort the list of paths lexicographically.
+ // FSO Depth-First Search order evaluates directories in lexicographical order
+ // (since it retrieves entries from RocksDB sorted by name within the same parent).
+ // Standard string sort on logical paths separated by "/" perfectly matches this DFS order.
+ List<RuleListWithDirectoryList> sortedConsolidatedRules =
+ consolidatedRules.stream().sorted(new RuleListWithDirectoryListOrder()).collect(Collectors.toList());
+
+ LOG.info("Final consolidated rules: " +
+ sortedConsolidatedRules.stream().map(RuleListWithDirectoryList::toString).collect(Collectors.joining(", ")));
+ if (test) {
+ consolidatedRuleList = sortedConsolidatedRules;
+ }
+ return sortedConsolidatedRules;
+ }
+
+ private boolean canSkipDir(String currentDirPath, String lastScannedDirInState) {
+ if (lastScannedDirInState == null || currentDirPath.isEmpty()) {
+ return false;
+ }
+ String[] cur = currentDirPath.split(OM_KEY_PREFIX);
+ String[] last = lastScannedDirInState.split(OM_KEY_PREFIX);
+ int n = Math.min(cur.length, last.length);
+ for (int i = 0; i < n; i++) {
+ int cmp = cur[i].compareTo(last[i]);
+ if (cmp != 0) {
+ // current name > last name -> skip
+ return cmp > 0;
+ }
}
+ return false;
}
@SuppressWarnings({"checkstyle:parameternumber", "checkstyle:MethodLength"})
private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table<String, OmKeyInfo> keyTable,
- String directoryPath, @Nullable OmDirectoryInfo dir, List<OmLCRule> ruleList, LimitedExpiredObjectList keyList,
- LimitedExpiredObjectList dirList) {
+ String directoryPath, @Nullable OmDirectoryInfo dir, String dirKey, List<OmLCRule> ruleList,
+ LimitedExpiredObjectList keyList, LimitedExpiredObjectList dirList,
+ OmLifecycleScanState.Builder scanStateBuilder) {
String volumeName = bucket.getVolumeName();
String bucketName = bucket.getBucketName();
LimitedSizeStack stack = new LimitedSizeStack(cachedDirMaxCount);
+ String lastScannedDirInState = scanStateBuilder == null ? null : scanStateBuilder.getLastScannedDir();
+ String lastScannedDirKeyInState = scanStateBuilder == null ? null : scanStateBuilder.getLastScannedDirKey();
+ String lastScannedKeyInState = scanStateBuilder == null ? null : scanStateBuilder.getLastScannedKey();
try {
if (dir != null) {
- stack.push(new PendingEvaluateDirectory(dir, directoryPath, null));
+ stack.push(new PendingEvaluateDirectory(dir, dirKey, directoryPath, null));
} else {
// put a placeholder PendingEvaluateDirectory to stack for bucket
- stack.push(new PendingEvaluateDirectory(null, "", null));
+ stack.push(new PendingEvaluateDirectory(null, "", "", null));
}
} catch (CapacityFullException e) {
LOG.warn("Abort evaluate {}/{} at {}", volumeName, bucketName, directoryPath != null ? directoryPath : "", e);
@@ -518,10 +679,54 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table
getOzoneManager() != null ? getOzoneManager().isLeaderReady() : "N/A");
return;
}
+
PendingEvaluateDirectory item = stack.pop();
OmDirectoryInfo currentDir = item.getDirectoryInfo();
String currentDirPath = item.getDirPath();
long currentDirObjID = currentDir == null ? bucket.getObjectID() : currentDir.getObjectID();
+ String currentDirTableKey = item.getDirTableKey();
+
+ /**
+ * /
+ * dir1 dir2 dir3 dir30
+ * / \ / \
+ * dir4 dir5 dir6 dir7
+ * / \
+ * dir8 dir9
+ * lastScannedDir = dir3/dir6/dir8, which means
+ * Scanned:
+ * dir30
+ * dir3/dir7
+ * dir3/dir6/dir9
+ * Half scanned:
+ * dir3/dir6/dir8
+ * Not scanned:
+ * dir3/dir6
+ * dir3
+ * dir1/dir5
+ * dir/dir4
+ * dir1
+ * directoryTable table key format : /volumeId/bucketId/parentId/dirName
+ * based on the depth first evaluation order, and stack push posh iteration pattern
+ * - dir1, on grand level of lastScannedDir, and name order < lastScannedDir grand, not scanned
+ * - dir2, on grand level of lastScannedDir, and name order < lastScannedDir grand, not scanned
+ * - dir3, grand of lastScannedDir, not scanned
+ * - dir30, on grand level of lastScannedDir, and name order > lastScannedDir grand, scanned, skip
+ * - dir3/dir6, parent of lastScannedDir, not scanned
+ * - dir3/dir7, parent level of lastScannedDir, and name order > lastScannedDir parent, scanned, skip
+ * - dir3/dir8, lastScannedDir, partially scanned,
+ * - dir3/dir9, same the same parentID as lastScannedDir, and name order > lastScannedDir, scanned, skip
+ */
+ if (canSkipDir(currentDirPath, lastScannedDirInState)) {
+ LOG.info("Skip {} in LifecycleActionTask for bucket {}. ", currentDirPath, bucketName);
+ continue;
+ }
+
+ lastScannedDir = currentDirPath;
+ lastScannedDirKey = currentDirTableKey;
+ if (shouldSaveState()) {
+ flushAndSaveState(bucket, keyList, dirList, scanStateBuilder);
+ }
// use current directory's object ID to iterate the keys and directories under it
String prefix =
@@ -529,10 +734,10 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table
LOG.debug("Prefix {} for {}/{}", prefix, bucket.getVolumeName(), bucket.getBucketName());
// get direct sub directories
- SubDirectorySummary subDirSummary = item.getSubDirSummary();
+ DirectoryList subDirSummary;
boolean newSubDirPushed = false;
long deletedDirCount = 0;
- if (subDirSummary == null) {
+ if (item.isFirstEvaluate()) {
try {
subDirSummary = getSubDirectory(currentDirObjID, prefix, omMetadataManager);
} catch (IOException e) {
@@ -540,7 +745,17 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table
LOG.warn("Failed to get sub directories of {} under {}/{}", currentDirPath, volumeName, bucketName, e);
continue;
}
+ } else {
+ // this item is a parent directory, check how many sub directories are deleted.
+ subDirSummary = item.getSubDirSummary();
+ for (OmDirectoryInfo subDir : subDirSummary.getSubDirList()) {
+ if (deletedDirSet.remove(subDir.getObjectID())) {
+ deletedDirCount++;
+ }
+ }
+ }
+ if (item.isFirstEvaluate()) {
// filter sub directory list
if (!subDirSummary.getSubDirList().isEmpty()) {
Iterator<OmDirectoryInfo> iterator = subDirSummary.getSubDirList().iterator();
@@ -550,6 +765,7 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table
currentDirPath + OM_KEY_PREFIX + subDir.getName();
if (subDirPath.startsWith(TRASH_PREFIX)) {
iterator.remove();
+ continue;
}
boolean matched = false;
for (OmLCRule rule : ruleList) {
@@ -565,7 +781,8 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table
}
if (!subDirSummary.getSubDirList().isEmpty()) {
- item.setSubDirSummary(subDirSummary);
+ item.setDirectoryList(subDirSummary);
+ item.setFirstEvaluate(false);
try {
stack.push(item);
} catch (CapacityFullException e) {
@@ -574,11 +791,13 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table
}
// depth first evaluation, push subDirs into stack
- for (OmDirectoryInfo subDir : subDirSummary.getSubDirList()) {
+ for (int i = 0; i < subDirSummary.getSubDirCount(); i++) {
+ OmDirectoryInfo subDir = subDirSummary.getSubDirList().get(i);
String subDirPath = currentDirPath.isEmpty() ? subDir.getName() :
currentDirPath + OM_KEY_PREFIX + subDir.getName();
try {
- stack.push(new PendingEvaluateDirectory(subDir, subDirPath, null));
+ stack.push(new PendingEvaluateDirectory(subDir, subDirSummary.getSubDirKeyList().get(i),
+ subDirPath, null));
} catch (CapacityFullException e) {
LOG.warn("Abort evaluate {}/{} at {}", volumeName, bucketName, subDirPath, e);
return;
@@ -586,13 +805,6 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table
}
newSubDirPushed = true;
}
- } else {
- // this item is a parent directory, check how many sub directories are deleted.
- for (OmDirectoryInfo subDir : subDirSummary.getSubDirList()) {
- if (deletedDirSet.remove(subDir.getObjectID())) {
- deletedDirCount++;
- }
- }
}
if (newSubDirPushed) {
@@ -628,39 +840,57 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table
// mark key as expired, check next key
if (keyList.isFull()) {
// if keyList is full, send delete/rename request for expired keys
- handleAndClearFullList(bucket, keyList, false);
+ handleAndClearFullList(bucket, keyList, false, scanStateBuilder, false);
}
keyList.add(keyPath, key.getReplicatedSize(), key.getUpdateID());
numKeysExpired++;
break;
}
}
+ lastScannedKey = entry.getKey().getCacheKey();
}
}
try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> keyTblItr =
keyTable.iterator(prefix)) {
+ boolean seekPerformed = false;
+ if (lastScannedDirKeyInState != null && lastScannedDirKeyInState.compareTo(currentDirTableKey) == 0 &&
+ lastScannedKeyInState != null && lastScannedKeyInState.startsWith(prefix)) {
+ LOG.info("Seek to key {} under directory {}", scanStateBuilder.getLastScannedKey(), lastScannedDirInState);
+ keyTblItr.seek(scanStateBuilder.getLastScannedKey());
+ seekPerformed = true;
+ }
+
while (keyTblItr.hasNext()) {
+ if (shouldSaveState()) {
+ LOG.info("Saving scan state for bucket {} at key {}", bucketName, lastScannedKey);
+ flushAndSaveState(bucket, keyList, dirList, scanStateBuilder);
+ }
Table.KeyValue<String, OmKeyInfo> keyValue = keyTblItr.next();
OmKeyInfo key = keyValue.getValue();
String keyPath = currentDirPath.isEmpty() ? key.getKeyName() :
currentDirPath + OM_KEY_PREFIX + key.getKeyName();
- numKeyIterated++;
+ if (seekPerformed && keyValue.getKey().equals(scanStateBuilder.getLastScannedKey())) {
+ continue;
+ }
if (deletedKeySetInCache.remove(keyValue.getKey()) || keySetInCache.remove(keyValue.getKey())) {
continue;
}
+ numKeyIterated++;
numKeysUnderDir++;
for (OmLCRule rule : ruleList) {
if (key.getParentObjectID() == currentDirObjID && rule.match(key, keyPath)) {
// mark key as expired, check next key
if (keyList.isFull()) {
// if keyList is full, send delete request for pending deletion keys
- handleAndClearFullList(bucket, keyList, false);
+ handleAndClearFullList(bucket, keyList, false, scanStateBuilder, false);
}
keyList.add(keyPath, key.getReplicatedSize(), key.getUpdateID());
numKeysExpired++;
+ break;
}
}
+ lastScannedKey = keyValue.getKey();
}
} catch (IOException e) {
// log failure and continue the process other directories in stack
@@ -671,25 +901,49 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table
// if this directory is empty or all files/subDirs are expired, evaluate itself
if ((numKeysUnderDir == 0 && subDirSummary.getSubDirCount() == 0) ||
(numKeysUnderDir == numKeysExpired && deletedDirCount == subDirSummary.getSubDirCount())) {
- for (OmLCRule rule : ruleList) {
- String path = (rule.getEffectivePrefix() != null && rule.getEffectivePrefix().endsWith(OM_KEY_PREFIX)) ?
+ List<String> pathList = new ArrayList<>();
+ boolean skipDir = false;
+ for (int i = 0; i < ruleList.size(); i++) { // NOPMD
+ OmLCRule rule = ruleList.get(i);
+ String path = rule.getEffectivePrefix() != null && rule.getEffectivePrefix().endsWith(OM_KEY_PREFIX) ?
currentDirPath + OM_KEY_PREFIX : currentDirPath;
+ if (path != null && path.equals(rule.getEffectivePrefix())) {
+ LOG.info("Prefix directory {} doesn't get expired", path);
+ skipDir = true;
+ break;
+ }
+ pathList.add(path);
+ }
+
+ if (skipDir) {
+ continue;
+ }
+ for (int i = 0; i < ruleList.size(); i++) {
+ String path = pathList.get(i);
+ OmLCRule rule = ruleList.get(i);
if (currentDir != null && rule.match(currentDir, path)) {
if (dirList.isFull()) {
- // if expiredDirList is full, send delete request for pending deletion directories
- handleAndClearFullList(bucket, dirList, true);
+ // if expiredDirList is full, send delete request for both pending deletion keys and directories
+ handleAndClearFullList(bucket, keyList, false, scanStateBuilder, false);
+ handleAndClearFullList(bucket, dirList, true, scanStateBuilder, false);
+ if (getInjector(2) != null && getInjector(2).getException() != null) {
+ Throwable ex = getInjector(2).getException();
+ getInjector(2).setException(null);
+ throw new RuntimeException(ex);
+ }
}
dirList.add(currentDirPath, 0, currentDir.getUpdateID());
deletedDirSet.add(currentDir.getObjectID());
+ break;
}
}
}
}
}
- private SubDirectorySummary getSubDirectory(long dirObjID, String prefix, OMMetadataManager metaMgr)
+ private DirectoryList getSubDirectory(long dirObjID, String prefix, OMMetadataManager metaMgr)
throws IOException {
- SubDirectorySummary subDirList = new SubDirectorySummary();
+ DirectoryList subDirList = new DirectoryList();
// Check all dirTable cache for any sub paths.
Table dirTable = metaMgr.getDirectoryTable();
@@ -706,7 +960,7 @@ private SubDirectorySummary getSubDirectory(long dirObjID, String prefix, OMMeta
continue;
}
if (cacheOmDirInfo.getParentObjectID() == dirObjID) {
- subDirList.addSubDir(cacheOmDirInfo);
+ subDirList.addSubDir(entry.getKey().getCacheKey(), cacheOmDirInfo, cacheOmDirInfo.getName());
}
}
@@ -721,46 +975,69 @@ private SubDirectorySummary getSubDirectory(long dirObjID, String prefix, OMMeta
continue;
}
if (dir.getParentObjectID() == dirObjID) {
- subDirList.addSubDir(dir);
+ subDirList.addSubDir(entry.getKey(), dir, dir.getName());
}
}
}
return subDirList;
}
+ private void flushAndSaveState(OmBucketInfo bucket, LimitedExpiredObjectList expiredKeyList,
+ LimitedExpiredObjectList expiredDirList, OmLifecycleScanState.Builder scanStateBuilder) {
+ boolean saved = false;
+ if (expiredKeyList != null && !expiredKeyList.isEmpty()) {
+ if (bucket.getBucketLayout() == OBJECT_STORE) {
+ sendDeleteKeysRequestAndClearList(bucket.getVolumeName(), bucket.getBucketName(), expiredKeyList,
+ false, scanStateBuilder, false);
+ } else {
+ handleAndClearFullList(bucket, expiredKeyList, false, scanStateBuilder, false);
+ }
+ saved = true;
+ }
+ if (expiredDirList != null && !expiredDirList.isEmpty()) {
+ if (bucket.getBucketLayout() != OBJECT_STORE) {
+ handleAndClearFullList(bucket, expiredDirList, true, scanStateBuilder, false);
+ saved = true;
+ }
+ }
+ if (!saved) {
+ sendSaveScanStateRequest(scanStateBuilder, false);
+ }
+ lastStateSaveTime = Time.monotonicNow();
+ lastStateSaveKeyCount = numKeyIterated;
+ }
+
private void evaluateBucket(OmBucketInfo bucketInfo,
- Table<String, OmKeyInfo> keyTable, List<OmLCRule> ruleList, LimitedExpiredObjectList expiredKeyList) {
+ Table<String, OmKeyInfo> keyTable, List<OmLCRule> ruleList, LimitedExpiredObjectList expiredKeyList,
+ OmLifecycleScanState.Builder scanStateBuilder) {
String volumeName = bucketInfo.getVolumeName();
String bucketName = bucketInfo.getBucketName();
+ String bucketPrefix = omMetadataManager.getBucketKey(volumeName, bucketName);
- // use bucket name as key iterator prefix
try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> keyTblItr =
- keyTable.iterator(omMetadataManager.getBucketKey(volumeName, bucketName))) {
+ keyTable.iterator(bucketPrefix)) {
+ boolean seekPerformed = false;
+ if (scanStateBuilder != null && scanStateBuilder.getLastScannedKey() != null) {
+ keyTblItr.seek(scanStateBuilder.getLastScannedKey());
+ seekPerformed = true;
+ }
+
while (keyTblItr.hasNext()) {
if (!shouldRun()) {
LOG.info("KeyLifecycleService is suspended or disabled. " +
"Stopping LifecycleActionTask for bucket {}.", bucketName);
return;
}
+ if (shouldSaveState()) {
+ flushAndSaveState(bucketInfo, expiredKeyList, null, scanStateBuilder);
+ }
Table.KeyValue<String, OmKeyInfo> keyValue = keyTblItr.next();
- OmKeyInfo key = keyValue.getValue();
- numKeyIterated++;
- if (bucketInfo.getBucketLayout() == BucketLayout.LEGACY &&
- key.getKeyName().startsWith(TRASH_PREFIX + OzoneConsts.OM_KEY_PREFIX)) {
- LOG.info("Skip evaluate trash directory {} and all its child files and sub directories", TRASH_PREFIX);
+ if (seekPerformed && keyValue.getKey().equals(scanStateBuilder.getLastScannedKey())) {
continue;
}
- for (OmLCRule rule : ruleList) {
- if (rule.match(key)) {
- // mark key as expired, check next key
- if (expiredKeyList.isFull()) {
- // if expiredKeyList is full, send delete/rename request for expired keys
- handleAndClearFullList(bucketInfo, expiredKeyList, false);
- }
- expiredKeyList.add(key.getKeyName(), key.getReplicatedSize(), key.getUpdateID());
- break;
- }
- }
+ processKey(bucketInfo, keyValue.getValue(), ruleList, expiredKeyList, scanStateBuilder);
+ numKeyIterated++;
+ lastScannedKey = keyValue.getKey();
}
} catch (IOException e) {
// log failure and continue the process to delete/move files already identified in this run
@@ -768,6 +1045,30 @@ private void evaluateBucket(OmBucketInfo bucketInfo,
}
}
+ private void processKey(OmBucketInfo bucketInfo, OmKeyInfo key, List<OmLCRule> ruleList,
+ LimitedExpiredObjectList expiredKeyList, OmLifecycleScanState.Builder scanStateBuilder) {
+ if (bucketInfo.getBucketLayout() == BucketLayout.LEGACY &&
+ key.getKeyName().startsWith(TRASH_PREFIX + OzoneConsts.OM_KEY_PREFIX)) {
+ return;
+ }
+ for (OmLCRule rule : ruleList) {
+ if (rule.match(key)) {
+ // mark key as expired, check next key
+ if (expiredKeyList.isFull()) {
+ // if expiredKeyList is full, send delete/rename request for expired keys
+ handleAndClearFullList(bucketInfo, expiredKeyList, false, scanStateBuilder, false);
+ if (getInjector(2) != null && getInjector(2).getException() != null) {
+ Throwable ex = getInjector(2).getException();
+ getInjector(2).setException(null);
+ throw new RuntimeException(ex);
+ }
+ }
+ expiredKeyList.add(key.getKeyName(), key.getReplicatedSize(), key.getUpdateID());
+ break;
+ }
+ }
+ }
+
/**
* Process AbortIncompleteMultipartUpload actions for incomplete multipart uploads.
* Iterates through the multipartInfoTable and aborts uploads that match the rule criteria
@@ -792,7 +1093,6 @@ private void processMultipartUploads(OmBucketInfo bucketInfo, List<OmLCRule> rul
"Stopping multipart upload processing for bucket {}.", bucketName);
return;
}
-
Table.KeyValue<String, OmMultipartKeyInfo> entry = mpuIterator.next();
OmMultipartKeyInfo mpuKeyInfo = entry.getValue();
numMultipartUploadIterated++;
@@ -961,41 +1261,47 @@ private void abortExpiredMultipartUploadsAndClear(OmBucketInfo bucketInfo,
}
/**
- * If prefix is /dir1/dir2, but dir1 doesn't exist, then it will return exception.
- * If prefix is /dir1/dir2, but dir2 doesn't exist, then it will return a list with dir1 only.
- * If prefix is /dir1/dir2, although dir1 exists, but get(dir1) failed with IOException,
- * then it will return exception too.
+ * If the prefix is /dir1/dir2, but dir1 doesn't exist, then it will return an exception.
+ * If the prefix is /dir1/dir2, but dir2 doesn't exist, then it will return a list with dir1 only.
+ * If the prefix is /dir1/dir2, although dir1 exists, but get(dir1) failed with IOException,
+ * then it will return an exception too.
*/
- private List<OmDirectoryInfo> getDirList(OmVolumeArgs volume, OmBucketInfo bucket, String prefix, String bucketKey)
+ private DirectoryList getDirList(long volumeID, OmBucketInfo bucket, String prefix, String bucketKey)
throws IOException {
- // find KeyInfo of each directory for prefix
+ // find KeyInfo of each directory for the prefix
java.nio.file.Path keyPath = Paths.get(prefix);
Iterator<java.nio.file.Path> elements = keyPath.iterator();
long lastKnownParentId = bucket.getObjectID();
- List<OmDirectoryInfo> dirList = new ArrayList<>();
+ DirectoryList directoryList = new DirectoryList();
+ StringBuffer currentDirPath = new StringBuffer();
while (elements.hasNext()) {
String dirName = elements.next().toString();
String dbDirName = omMetadataManager.getOzonePathKey(
- volume.getObjectID(), bucket.getObjectID(),
- lastKnownParentId, dirName);
+ volumeID, bucket.getObjectID(), lastKnownParentId, dirName);
try {
OmDirectoryInfo omDirInfo = omMetadataManager.getDirectoryTable().get(dbDirName);
- // It's OK there is no directory for the last part of prefix, which is probably not a directory
+ // It's OK there is no directory for the last part of the prefix, which is probably not a directory
if (omDirInfo == null) {
if (elements.hasNext()) {
throw new OMException("Directory " + dbDirName + " does not exist for bucket " + bucketKey,
OMException.ResultCodes.DIRECTORY_NOT_FOUND);
}
+ directoryList.setNotAllResolvedPrefix();
} else {
- dirList.add(omDirInfo);
+ if (currentDirPath.length() == 0) {
+ currentDirPath.append(dirName);
+ } else {
+ currentDirPath.append('/').append(dirName);
+ }
+ directoryList.addSubDir(dbDirName, omDirInfo, currentDirPath.toString());
lastKnownParentId = omDirInfo.getObjectID();
}
} catch (IOException e) {
- LOG.warn("Failed to get directory {} from bucket {}", dbDirName, bucketKey, e);
- throw new IOException("Failed to get directory " + dbDirName + " from bucket " + bucketKey);
+ LOG.warn("Failed to get directory {} for bucket {}", dbDirName, bucketKey, e);
+ throw new IOException("Failed to get directory " + dbDirName + " for bucket " + bucketKey);
}
}
- return dirList;
+ return directoryList;
}
private void onFailure(String bucketName) {
@@ -1004,6 +1310,12 @@ private void onFailure(String bucketName) {
metrics.incNumKeyIterated(numKeyIterated);
metrics.incNumDirIterated(numDirIterated);
metrics.incNumMultipartUploadIterated(numMultipartUploadIterated);
+ long timeSpent = Time.monotonicNow() - taskStartTime;
+ LOG.info("Spent {} ms on bucket {} to iterate {} keys and {} dirs and {} multipart uploads, " +
+ "deleted {} keys with {} bytes, and {} dirs, renamed {} keys with {} bytes, and {} dirs to trash, " +
+ "aborted {} multipart uploads", timeSpent, bucketName, numKeyIterated,
+ numDirIterated, numMultipartUploadIterated, numKeyDeleted, sizeKeyDeleted, numDirDeleted,
+ numKeyRenamed, sizeKeyRenamed, numDirRenamed, numMultipartUploadAborted);
}
private void onSuccess(String bucketName) {
@@ -1015,23 +1327,62 @@ private void onSuccess(String bucketName) {
metrics.incNumDirIterated(numDirIterated);
metrics.incNumMultipartUploadIterated(numMultipartUploadIterated);
metrics.incNumMultipartUploadAborted(numMultipartUploadAborted);
- LOG.info("Spend {} ms on bucket {} to iterate {} keys and {} dirs and {} multipart uploads, " +
+ LOG.info("Spent {} ms on bucket {} to iterate {} keys and {} dirs and {} multipart uploads, " +
"deleted {} keys with {} bytes, and {} dirs, renamed {} keys with {} bytes, and {} dirs to trash, " +
"aborted {} multipart uploads", timeSpent, bucketName, numKeyIterated,
numDirIterated, numMultipartUploadIterated, numKeyDeleted, sizeKeyDeleted, numDirDeleted,
numKeyRenamed, sizeKeyRenamed, numDirRenamed, numMultipartUploadAborted);
}
- private void handleAndClearFullList(OmBucketInfo bucket, LimitedExpiredObjectList keysList, boolean dir) {
+ private void handleAndClearFullList(OmBucketInfo bucket, LimitedExpiredObjectList keysList,
+ boolean dir, OmLifecycleScanState.Builder scanStateBuilder, boolean scanFinished) {
if (moveToTrashEnabled.get() && bucket.getBucketLayout() != OBJECT_STORE && ozoneTrash != null) {
moveToTrash(bucket, keysList, dir);
+ sendSaveScanStateRequest(scanStateBuilder, scanFinished);
} else {
- sendDeleteKeysRequestAndClearList(bucket.getVolumeName(), bucket.getBucketName(), keysList, dir);
+ sendDeleteKeysRequestAndClearList(bucket.getVolumeName(), bucket.getBucketName(), keysList, dir,
+ scanStateBuilder, scanFinished);
}
}
- private void sendDeleteKeysRequestAndClearList(String volume, String bucket,
- LimitedExpiredObjectList keysList, boolean dir) {
+ private void sendSaveScanStateRequest(OmLifecycleScanState.Builder scanStateBuilder, boolean scanFinished) {
+ if (scanStateBuilder != null) {
+ if (lastScannedDir != null) {
+ scanStateBuilder.setLastScannedDir(lastScannedDir);
+ }
+ if (lastScannedDirKey != null) {
+ scanStateBuilder.setLastScannedDirKey(lastScannedDirKey);
+ }
+ if (lastScannedKey != null) {
+ scanStateBuilder.setLastScannedKey(lastScannedKey);
+ }
+ if (scanFinished) {
+ scanStateBuilder.setScanEndTime(System.currentTimeMillis());
+ }
+ OmLifecycleScanState state = scanStateBuilder.build();
+
+ SaveLifecycleScanStateRequest saveRequest = SaveLifecycleScanStateRequest.newBuilder()
+ .setState(state.getProtobuf())
+ .build();
+
+ OMRequest omRequest = OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.SaveLifecycleScanState)
+ .setVersion(ClientVersion.CURRENT_VERSION)
+ .setClientId(clientId.toString())
+ .setSaveLifecycleScanStateRequest(saveRequest)
+ .build();
+
+ LOG.debug("Save scan state {}", state);
+ try {
+ OzoneManagerRatisUtils.submitRequest(getOzoneManager(), omRequest, clientId, callId.getAndIncrement());
+ } catch (ServiceException e) {
+ LOG.error("Failed to submit SaveLifecycleScanState request", e);
+ }
+ }
+ }
+
+ private void sendDeleteKeysRequestAndClearList(String volume, String bucket, LimitedExpiredObjectList keysList,
+ boolean dir, OmLifecycleScanState.Builder scanStateBuilder, boolean scanFinished) {
try {
if (getInjector(1) != null) {
try {
@@ -1053,10 +1404,29 @@ private void sendDeleteKeysRequestAndClearList(String volume, String bucket,
builder.addAllUpdateIDs(keysList.updateIDSubList(startIndex, endIndex));
DeleteKeyArgs deleteKeyArgs = builder.build();
- DeleteKeysRequest deleteKeysRequest = DeleteKeysRequest.newBuilder()
+ DeleteKeysRequest.Builder requestBuilder = DeleteKeysRequest.newBuilder()
.setDeleteKeys(deleteKeyArgs)
- .setSourceType(RequestSource.LIFECYCLE)
- .build();
+ .setSourceType(RequestSource.LIFECYCLE);
+
+ if (scanStateBuilder != null) {
+ if (lastScannedKey != null) {
+ scanStateBuilder.setLastScannedKey(lastScannedKey);
+ }
+ if (lastScannedDir != null) {
+ scanStateBuilder.setLastScannedDir(lastScannedDir);
+ }
+ if (lastScannedDirKey != null) {
+ scanStateBuilder.setLastScannedDirKey(lastScannedDirKey);
+ }
+ if (scanFinished) {
+ scanStateBuilder.setScanEndTime(System.currentTimeMillis());
+ }
+ OmLifecycleScanState state = scanStateBuilder.build();
+ requestBuilder.setScanState(state.getProtobuf());
+ LOG.debug("Save scan state: {}", state);
+ }
+
+ DeleteKeysRequest deleteKeysRequest = requestBuilder.build();
LOG.debug("request size {} for {} keys", deleteKeysRequest.getSerializedSize(), keyCount);
if (deleteKeysRequest.getSerializedSize() < ratisByteLimit) {
@@ -1323,6 +1693,12 @@ public void add(String name, long size, long updateID) {
objectUpdateIDs.add(updateID);
}
+ public void addAll(LimitedExpiredObjectList other) {
+ objectNames.addAll(other.objectNames);
+ objectReplicatedSize.addAll(other.objectReplicatedSize);
+ objectUpdateIDs.addAll(other.objectUpdateIDs);
+ }
+
public int size() {
return objectNames.size();
}
@@ -1395,6 +1771,10 @@ public void add(T element) {
internalList.add(element);
}
+ public void addAll(LimitedSizeList<T> other) {
+ internalList.addAll(other.internalList);
+ }
+
public T get(int index) {
return internalList.get(index);
}
@@ -1484,13 +1864,25 @@ public void clear() {
*/
public static class PendingEvaluateDirectory {
private final OmDirectoryInfo directoryInfo;
- private final String dirPath;
- private SubDirectorySummary subDirSummary;
+ private String dirTableKey;
+ private String dirPath;
+ private DirectoryList directoryList;
+ private boolean firstEvaluate;
- public PendingEvaluateDirectory(OmDirectoryInfo dir, String path, SubDirectorySummary summary) {
+ public PendingEvaluateDirectory(OmDirectoryInfo dir, String dirTableKey, String dirPath, DirectoryList summary) {
this.directoryInfo = dir;
- this.dirPath = path;
- this.subDirSummary = summary;
+ this.dirTableKey = dirTableKey;
+ this.dirPath = dirPath;
+ this.directoryList = summary;
+ this.firstEvaluate = true;
+ }
+
+ public String getDirTableKey() {
+ return dirTableKey;
+ }
+
+ public void setDirTableKey(String tableKey) {
+ dirTableKey = tableKey;
}
public String getDirPath() {
@@ -1501,28 +1893,41 @@ public OmDirectoryInfo getDirectoryInfo() {
return directoryInfo;
}
- public SubDirectorySummary getSubDirSummary() {
- return subDirSummary;
+ public DirectoryList getSubDirSummary() {
+ return directoryList;
}
- public void setSubDirSummary(SubDirectorySummary summary) {
- subDirSummary = summary;
+ public void setDirectoryList(DirectoryList summary) {
+ directoryList = summary;
+ }
+
+ public boolean isFirstEvaluate() {
+ return firstEvaluate;
+ }
+
+ public void setFirstEvaluate(boolean firstEvaluate) {
+ this.firstEvaluate = firstEvaluate;
}
}
/**
- * An in-memory class to hold sub directory summary.
+ * An in-memory class to hold a directory list.
*/
- public static class SubDirectorySummary {
+ public static class DirectoryList {
private final List<OmDirectoryInfo> subDirList;
- private long subDirCount;
+ private final List<String> subDirKeyList;
+ private final List<String> subDirKeyPathList;
+ private int subDirCount;
+ private boolean allResolvedPrefix = true;
- public SubDirectorySummary() {
+ public DirectoryList() {
this.subDirList = new ArrayList<>();
+ this.subDirKeyList = new ArrayList<>();
+ this.subDirKeyPathList = new ArrayList<>();
this.subDirCount = 0;
}
- public long getSubDirCount() {
+ public int getSubDirCount() {
return subDirCount;
}
@@ -1530,10 +1935,149 @@ public List<OmDirectoryInfo> getSubDirList() {
return subDirList;
}
- public void addSubDir(OmDirectoryInfo dir) {
+ public List<String> getSubDirKeyList() {
+ return subDirKeyList;
+ }
+
+ public void addSubDir(String key, OmDirectoryInfo dir, String dirPath) {
+ subDirKeyList.add(key);
subDirList.add(dir);
+ subDirKeyPathList.add(dirPath);
subDirCount++;
}
+
+ public OmDirectoryInfo getLastSubDir() {
+ return subDirCount > 0 ? subDirList.get(subDirCount - 1) : null;
+ }
+
+ public String getLastSubDirKey() {
+ return subDirCount > 0 ? subDirKeyList.get(subDirCount - 1) : null;
+ }
+
+ public String getLastSubDirPath() {
+ return subDirCount > 0 ? subDirKeyPathList.get(subDirCount - 1) : null;
+ }
+
+ public boolean isAllResolvedPrefix() {
+ return allResolvedPrefix;
+ }
+
+ public void setNotAllResolvedPrefix() {
+ this.allResolvedPrefix = false;
+ }
+
+ public boolean isEmpty() {
+ return subDirCount == 0;
+ }
+
+ @Override
+ public String toString() {
+ return "DirectoryList { " +
+ "subDirList = " + subDirList +
+ ", subDirKeyList = " + subDirKeyList +
+ ", subDirKeyPathList = " + subDirKeyPathList +
+ ", subDirCount = " + subDirCount +
+ ", allResolvedPrefix = " + allResolvedPrefix +
+ '}';
+ }
+ }
+
+ /**
+ * An in-memory class to hold a rule list, together with the resolved prefix's directory list.
+ */
+ public static class RuleListWithDirectoryList {
+ private DirectoryList dirList;
+ private final List<OmLCRule> ruleList;
+ private String consolidatedPrefix;
+
+ public RuleListWithDirectoryList() {
+ this.ruleList = new ArrayList<>();
+ this.dirList = new DirectoryList();
+ }
+
+ public RuleListWithDirectoryList(List<OmLCRule> ruleList, DirectoryList dirList, String prefix) {
+ this.ruleList = ruleList;
+ this.dirList = dirList;
+ this.consolidatedPrefix = prefix;
+ }
+
+ public List<OmLCRule> getRuleList() {
+ return ruleList;
+ }
+
+ public DirectoryList getDirList() {
+ return dirList;
+ }
+
+ public void addRule(OmLCRule rule) {
+ this.ruleList.add(rule);
+ }
+
+ public void setDirList(DirectoryList dirList) {
+ this.dirList = dirList;
+ }
+
+ public String getConsolidatedPrefix() {
+ return consolidatedPrefix;
+ }
+
+ public void setConsolidatedPrefix(String consolidatedPrefix) {
+ this.consolidatedPrefix = consolidatedPrefix;
+ }
+
+ public boolean isEmpty() {
+ return dirList.isEmpty() && ruleList.isEmpty() && consolidatedPrefix == null;
+ }
+
+ @Override
+ public String toString() {
+ return "RuleListWithDirectoryList { " +
+ "dirList = " + dirList +
+ ", ruleList = " + ruleList +
+ ", consolidatedPrefix = '" + consolidatedPrefix + '\'' +
+ '}';
+ }
+ }
+
+ /**
+ * Orders of RuleListWithDirectoryList.
+ */
+ public static class RuleListWithDirectoryListOrder implements Comparator<RuleListWithDirectoryList> {
+
+ public static final Comparator<RuleListWithDirectoryList> INSTANCE =
+ new RuleListWithDirectoryListOrder();
+
+ @Override
+ public int compare(RuleListWithDirectoryList o1, RuleListWithDirectoryList o2) {
+ List<OmDirectoryInfo> dirList1 = o1.getDirList().getSubDirList();
+ List<OmDirectoryInfo> dirList2 = o2.getDirList().getSubDirList();
+
+ // find their shared parent directory
+ OmDirectoryInfo parent = null;
+ for (int i = dirList1.size() - 1; i >= 0; i--) {
+ long objID = dirList1.get(i).getObjectID();
+ for (int j = dirList2.size() - 1; j >= 0; j--) {
+ if (dirList2.get(j).getObjectID() == objID) {
+ parent = dirList2.get(j);
+ break;
+ }
+ }
+ if (parent != null) {
+ break;
+ }
+ }
+ if (parent == null) {
+ // e.g, dir1 and dir2, dir2 should ahead of dir2, given the depth-first and dir's RocksDB key order
+ return dirList2.get(0).getName().compareTo(dirList1.get(0).getName());
+ } else {
+ long parentID = parent.getObjectID();
+ OmDirectoryInfo dir1 = dirList1.stream().filter(dir -> dir.getParentObjectID() == parentID)
+ .collect(Collectors.toList()).get(0);
+ OmDirectoryInfo dir2 = dirList2.stream().filter(dir -> dir.getParentObjectID() == parentID)
+ .collect(Collectors.toList()).get(0);
+ return dir2.getName().compareTo(dir1.getName());
+ }
+ }
}
/**
@@ -1572,4 +2116,16 @@ public CapacityFullException(String message) {
super(message);
}
}
+
+ public static void setTest(boolean test) {
+ KeyLifecycleService.test = test;
+ }
+
+ public static void reSetConsolidatedRuleList() {
+ consolidatedRuleList = null;
+ }
+
+ public static List<RuleListWithDirectoryList> getConsolidatedRuleList() {
+ return consolidatedRuleList;
+ }
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
index 41be905235b..799bd0ab3db 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
@@ -33,6 +33,7 @@
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.LIFECYCLE_CONFIGURATION_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.LIFECYCLE_SCAN_STATE_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.META_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.MULTIPART_INFO_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.MULTIPART_PARTS_TABLE;
@@ -140,7 +141,8 @@ public class TestOmMetadataManager {
SNAPSHOT_INFO_TABLE,
SNAPSHOT_RENAMED_TABLE,
COMPACTION_LOG_TABLE,
- LIFECYCLE_CONFIGURATION_TABLE
+ LIFECYCLE_CONFIGURATION_TABLE,
+ LIFECYCLE_SCAN_STATE_TABLE
};
private OMMetadataManager omMetadataManager;
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleSaveScanStateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleSaveScanStateRequest.java
new file mode 100644
index 00000000000..8c5a806ac94
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleSaveScanStateRequest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.request.lifecycle;
+
+import static org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager.maxLayoutVersion;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.UUID;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState;
+import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleScanState;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SaveLifecycleScanStateRequest;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests OMLifecycleSaveScanStateRequest.
+ */
+public class TestOMLifecycleSaveScanStateRequest {
+
+ @Test
+ public void testPreExecuteAdminCheck() throws Exception {
+ OzoneManager ozoneManager = mock(OzoneManager.class);
+ OMLayoutVersionManager versionManager = mock(OMLayoutVersionManager.class);
+ when(versionManager.getMetadataLayoutVersion()).thenReturn(maxLayoutVersion());
+ when(ozoneManager.getVersionManager()).thenReturn(versionManager);
+
+ // Test when ACLs are enabled but user is not admin
+ when(ozoneManager.getAclsEnabled()).thenReturn(true);
+ when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(false);
+
+ OMRequest omRequest = OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.SaveLifecycleScanState)
+ .setClientId(UUID.randomUUID().toString())
+ .setSaveLifecycleScanStateRequest(SaveLifecycleScanStateRequest.newBuilder()
+ .setState(LifecycleScanState.newBuilder().setBucketKey("dummy").setScanStartTime(1L).build())
+ .build())
+ .build();
+
+ OMLifecycleSaveScanStateRequest request = new OMLifecycleSaveScanStateRequest(omRequest);
+ request.setUGI(UserGroupInformation.getCurrentUser());
+
+ OMException exception = assertThrows(OMException.class, () -> {
+ request.preExecute(ozoneManager);
+ });
+
+ assertEquals(OMException.ResultCodes.ACCESS_DENIED, exception.getResult());
+ assertTrue(exception.getMessage().contains("Superuser privilege is required"));
+
+ // Test when user is admin
+ when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(true);
+ assertDoesNotThrow(() -> request.preExecute(ozoneManager));
+
+ // Test when ACLs are disabled
+ when(ozoneManager.getAclsEnabled()).thenReturn(false);
+ when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(false);
+ assertDoesNotThrow(() -> request.preExecute(ozoneManager));
+ }
+
+ @Test
+ public void testValidateAndUpdateCache() throws Exception {
+ OzoneManager ozoneManager = mock(OzoneManager.class);
+ OMMetadataManager omMetadataManager = mock(OMMetadataManager.class);
+ when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager);
+ Table<String, OmLifecycleScanState> table = mock(Table.class);
+ when(omMetadataManager.getLifecycleScanStateTable()).thenReturn(table);
+
+ LifecycleScanState stateProto = LifecycleScanState.newBuilder()
+ .setBucketKey("/vol1/bucket1")
+ .setScanStartTime(123456789L)
+ .setLastScannedKey("key1")
+ .build();
+
+ SaveLifecycleScanStateRequest saveRequest = SaveLifecycleScanStateRequest.newBuilder()
+ .setState(stateProto)
+ .build();
+
+ OMRequest omRequest = OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.SaveLifecycleScanState)
+ .setClientId(UUID.randomUUID().toString())
+ .setSaveLifecycleScanStateRequest(saveRequest)
+ .build();
+
+ OMLifecycleSaveScanStateRequest request = new OMLifecycleSaveScanStateRequest(omRequest);
+ OMClientResponse response = request.validateAndUpdateCache(ozoneManager, 100L);
+ assertNotNull(response);
+ assertEquals(OzoneManagerProtocolProtos.Status.OK, response.getOMResponse().getStatus());
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponse.java
index 229f4cb459b..0422aa0e63e 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponse.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponse.java
@@ -101,7 +101,7 @@ public void testKeysDeleteResponse() throws Exception {
protected OMClientResponse getOmKeysDeleteResponse(OMResponse omResponse,
OmBucketInfo omBucketInfo) {
return new OMKeysDeleteResponse(
- omResponse, omKeyInfoList, omBucketInfo, Collections.emptyMap());
+ omResponse, omKeyInfoList, omBucketInfo, Collections.emptyMap(), null);
}
@Test
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java
index 1f99c90e4d9..c34356e2d1c 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java
@@ -111,7 +111,7 @@ protected OMClientResponse getOmKeysDeleteResponse(OMResponse omResponse,
OmBucketInfo omBucketInfo) {
return new OMKeysDeleteResponseWithFSO(
omResponse, getOmKeyInfoList(), dirDeleteList, omBucketInfo,
- volId, Collections.emptyMap());
+ volId, Collections.emptyMap(), null);
}
@Test
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/lifecycle/TestOMLifecycleSaveScanStateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/lifecycle/TestOMLifecycleSaveScanStateResponse.java
new file mode 100644
index 00000000000..4109a9f0f74
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/lifecycle/TestOMLifecycleSaveScanStateResponse.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.response.lifecycle;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests OMLifecycleSaveScanStateResponse.
+ */
+public class TestOMLifecycleSaveScanStateResponse {
+
+ @Test
+ public void testAddToDBBatch() throws Exception {
+ OMMetadataManager omMetadataManager = mock(OMMetadataManager.class);
+ BatchOperation batchOperation = mock(BatchOperation.class);
+ Table<String, OmLifecycleScanState> table = mock(Table.class);
+ when(omMetadataManager.getLifecycleScanStateTable()).thenReturn(table);
+
+ OmLifecycleScanState state = new OmLifecycleScanState.Builder()
+ .setBucketKey("/vol1/bucket1")
+ .setScanStartTime(123456789L)
+ .setLastScannedKey("key1")
+ .build();
+
+ OMResponse omResponse = OMResponse.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.SaveLifecycleScanState)
+ .setStatus(OzoneManagerProtocolProtos.Status.OK)
+ .build();
+
+ OMLifecycleSaveScanStateResponse response = new OMLifecycleSaveScanStateResponse(omResponse, state);
+ response.addToDBBatch(omMetadataManager, batchOperation);
+
+ verify(table, times(1)).putWithBatch(batchOperation, "/vol1/bucket1", state);
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
index cd2191633ee..6b8f2e6e12e 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
@@ -27,6 +27,10 @@
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT;
import static org.apache.hadoop.ozone.om.OmConfig.Keys.ENABLE_FILESYSTEM_PATHS;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST;
import static org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED;
@@ -44,10 +48,17 @@
import static org.junit.jupiter.api.Assertions.fail;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
import static org.junit.jupiter.params.provider.Arguments.arguments;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.Mockito.atLeast;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
import com.google.common.collect.ImmutableMap;
import java.io.File;
import java.io.IOException;
+import java.lang.reflect.Field;
import java.security.PrivilegedExceptionAction;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
@@ -71,6 +82,7 @@
import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.hdds.server.ServerUtils;
import org.apache.hadoop.hdds.utils.db.DBConfigFromFile;
+import org.apache.hadoop.hdds.utils.db.RocksDatabaseException;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
@@ -79,6 +91,7 @@
import org.apache.hadoop.ozone.om.FaultInjectorImpl;
import org.apache.hadoop.ozone.om.KeyManager;
import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.OmTestManagers;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.OzoneTrash;
@@ -99,6 +112,7 @@
import org.apache.hadoop.ozone.om.helpers.OmLCRule;
import org.apache.hadoop.ozone.om.helpers.OmLifecycleConfiguration;
import org.apache.hadoop.ozone.om.helpers.OmLifecycleRuleAndOperator;
+import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState;
import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
@@ -125,6 +139,8 @@
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.Parameter;
+import org.junit.jupiter.params.ParameterizedClass;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.EnumSource;
@@ -146,6 +162,8 @@
*/
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
@Timeout(300)
+@ParameterizedClass
+@MethodSource("stateSaveConfiguration")
class TestKeyLifecycleService extends OzoneTestBase {
private static final Logger LOG =
LoggerFactory.getLogger(TestKeyLifecycleService.class);
@@ -168,6 +186,20 @@ class TestKeyLifecycleService extends OzoneTestBase {
private KeyLifecycleServiceMetrics metrics;
private long bucketObjectID;
+ @Parameter(0)
+ private long stateSaveInternal;
+
+ @Parameter(1)
+ private long maxKeysProcessedPerState;
+
+ static Stream<Arguments> stateSaveConfiguration() {
+ return Stream.of(
+ Arguments.of(-1, -1),
+ Arguments.of(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT,
+ OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT)
+ );
+ }
+
@BeforeAll
void setup() {
ExitUtils.disableSystemExit();
@@ -184,7 +216,10 @@ private void createConfig(File testDir) {
conf.setInt(OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE, 50);
conf.setQuietMode(false);
conf.setBoolean(ENABLE_FILESYSTEM_PATHS, false);
+ conf.setLong(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS, stateSaveInternal);
+ conf.setLong(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED, maxKeysProcessedPerState);
OmLCExpiration.setTest(true);
+ KeyLifecycleService.setTest(true);
}
private void createSubject() throws Exception {
@@ -221,6 +256,7 @@ void setup(@TempDir File testDir) throws Exception {
void resume() {
keyLifecycleService.setOzoneTrash(null);
keyLifecycleService.setMoveToTrashEnabled(true);
+ KeyLifecycleService.setInjectors(null);
}
@AfterAll
@@ -251,12 +287,13 @@ void testAllKeyExpired(BucketLayout bucketLayout, boolean createPrefix) throws I
TimeoutException, InterruptedException {
final String volumeName = getTestName();
final String bucketName = uniqueObjectName("bucket");
- String prefix = "key";
+ String keyPrefix = "key";
+ String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : "key";
long initialDeletedKeyCount = getDeletedKeyCount();
long initialKeyCount = getKeyCount(bucketLayout);
// create keys
List<OmKeyArgs> keyList =
- createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, prefix, null);
+ createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, keyPrefix, null);
// check there are keys in keyTable
assertEquals(KEY_COUNT, keyList.size());
GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == KEY_COUNT,
@@ -265,15 +302,510 @@ void testAllKeyExpired(BucketLayout bucketLayout, boolean createPrefix) throws I
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
if (createPrefix) {
- createLifecyclePolicy(volumeName, bucketName, bucketLayout, prefix, null, date.toString(), true);
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true);
} else {
- OmLCFilter.Builder filter = getOmLCFilterBuilder(prefix, null, null);
+ OmLCFilter.Builder filter = getOmLCFilterBuilder(rulePrefix, null, null);
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true);
+ }
+
+ GenericTestUtils.waitFor(() ->
+ (getDeletedKeyCount() - initialDeletedKeyCount) == KEY_COUNT, WAIT_CHECK_INTERVAL, 10000);
+ assertEquals(0, getKeyCount(bucketLayout) - initialKeyCount);
+ deleteLifecyclePolicy(volumeName, bucketName);
+ }
+
+ @ParameterizedTest
+ @MethodSource("parameters1")
+ void testScanStatePiggybackedOnDelete(BucketLayout bucketLayout, boolean createPrefix) throws IOException,
+ TimeoutException, InterruptedException {
+ final String volumeName = getTestName();
+ final String bucketName = uniqueObjectName("bucket");
+ String keyPrefix = "key";
+ String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : "key";
+ long initialDeletedKeyCount = getDeletedKeyCount();
+ long initialKeyCount = getKeyCount(bucketLayout);
+ // create keys
+ List<OmKeyArgs> keyList =
+ createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, keyPrefix, null);
+ // check there are keys in keyTable
+ assertEquals(KEY_COUNT, keyList.size());
+ GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == KEY_COUNT,
+ WAIT_CHECK_INTERVAL, 1000);
+ // create Lifecycle configuration
+ ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
+ ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
+
+ if (createPrefix) {
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true);
+ } else {
+ OmLCFilter.Builder filter = getOmLCFilterBuilder(rulePrefix, null, null);
createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true);
}
+ // Wait for deletion
GenericTestUtils.waitFor(() ->
(getDeletedKeyCount() - initialDeletedKeyCount) == KEY_COUNT, WAIT_CHECK_INTERVAL, 10000);
assertEquals(0, getKeyCount(bucketLayout) - initialKeyCount);
+
+ // Verify that scan state was updated through the DeleteKeysRequest
+ String bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
+ OmLifecycleScanState scanState = metadataManager.getLifecycleScanStateTable().get(bucketKey);
+ assertNotNull(scanState);
+ assertNotNull(scanState.getScanEndTime());
+
+ deleteLifecyclePolicy(volumeName, bucketName);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"FILE_SYSTEM_OPTIMIZED", "OBJECT_STORE"})
+ void testPeriodicStateSave(BucketLayout bucketLayout) throws Exception {
+ final String volumeName = getTestName();
+ final String bucketName = uniqueObjectName("bucket");
+ String keyPrefix = "key";
+ String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : "key";
+ long initialDeletedKeyCount = getDeletedKeyCount();
+ long initialKeyCount = getKeyCount(bucketLayout);
+ long initialNumKeyDeleted = metrics.getNumKeyDeleted().value();
+ long initialNumKeyIterated = metrics.getNumKeyIterated().value();
+ int testKeyCount = 5;
+
+ // Suspend service so it doesn't process immediately after we create the policy
+ keyLifecycleService.suspend();
+
+ // create keys
+ List<OmKeyArgs> keyList =
+ createKeys(volumeName, bucketName, bucketLayout, testKeyCount, 1, keyPrefix, null);
+ assertEquals(testKeyCount, keyList.size());
+ GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == testKeyCount,
+ WAIT_CHECK_INTERVAL, 1000);
+
+ // Inject spy to LifecycleScanStateTable to count put operations
+ Field tableField = OmMetadataManagerImpl.class.getDeclaredField("lifecycleScanStateTable");
+ tableField.setAccessible(true);
+ Table<String, OmLifecycleScanState> originalTable =
+ (Table<String, OmLifecycleScanState>) tableField.get(metadataManager);
+ Table<String, OmLifecycleScanState> spyTable = spy(originalTable);
+ tableField.set(metadataManager, spyTable);
+
+ try {
+ // create Lifecycle configuration
+ ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
+ ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true);
+
+ // Resume the service
+ keyLifecycleService.resume();
+
+ // Wait for deletion
+ GenericTestUtils.waitFor(() ->
+ (getDeletedKeyCount() - initialDeletedKeyCount) == testKeyCount, WAIT_CHECK_INTERVAL, 10000);
+ assertEquals(0, getKeyCount(bucketLayout) - initialKeyCount);
+
+ String bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
+
+ if (stateSaveInternal == -1) {
+ // With 5 keys and stateSaveIntervalMs = -1, it should save on every key iteration.
+ // It will save at least 5 times (one for each key).
+ verify(spyTable, atLeast(5))
+ .addCacheEntry(argThat(k -> k.getCacheKey().equals(bucketKey)), any());
+ } else {
+ // With 5 keys and maxKeysProcessedPerState = 100000, there is 1 save piggybacked in KeysDelete request.
+ verify(spyTable, atLeast(1))
+ .addCacheEntry(argThat(k -> k.getCacheKey().equals(bucketKey)), any());
+ }
+ GenericTestUtils.waitFor(() ->
+ (getDeletedKeyCount() - initialDeletedKeyCount) == testKeyCount, WAIT_CHECK_INTERVAL, 5000);
+ GenericTestUtils.waitFor(() ->
+ (getDeletedKeyCount() - initialDeletedKeyCount) == testKeyCount, WAIT_CHECK_INTERVAL, 5000);
+ GenericTestUtils.waitFor(() ->
+ (metrics.getNumKeyDeleted().value() - initialNumKeyDeleted) == testKeyCount, WAIT_CHECK_INTERVAL, 5000);
+ GenericTestUtils.waitFor(() ->
+ (metrics.getNumKeyIterated().value() - initialNumKeyIterated) == testKeyCount, WAIT_CHECK_INTERVAL, 5000);
+ } finally {
+ deleteLifecyclePolicy(volumeName, bucketName);
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("parameters1")
+ void testBucketScanResume(BucketLayout bucketLayout, boolean createPrefix) throws IOException,
+ TimeoutException, InterruptedException {
+ final String volumeName = getTestName();
+ final String bucketName = uniqueObjectName("bucket");
+ String keyPrefix = "key";
+ String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : "key";
+ long initialDeletedKeyCount = getDeletedKeyCount();
+ long initialKeyCount = getKeyCount(bucketLayout);
+ int testKeyCount = 5;
+
+ // Suspend service so it doesn't process immediately after we create the policy
+ keyLifecycleService.suspend();
+
+ // create keys
+ List<OmKeyArgs> keyList =
+ createKeys(volumeName, bucketName, bucketLayout, testKeyCount, 1, keyPrefix, null);
+ assertEquals(testKeyCount, keyList.size());
+ GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == testKeyCount,
+ WAIT_CHECK_INTERVAL, 1000);
+
+ // determine db keys
+ List<String> dbKeys = new ArrayList<>();
+ long bucketId =
+ metadataManager.getBucketTable().get(metadataManager.getBucketKey(volumeName, bucketName)).getObjectID();
+ long volumeId = metadataManager.getVolumeTable().get(metadataManager.getVolumeKey(volumeName)).getObjectID();
+
+ for (OmKeyArgs args : keyList) {
+ if (bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
+ dbKeys.add(metadataManager.getOzonePathKey(volumeId, bucketId, bucketId, args.getKeyName()));
+ } else {
+ dbKeys.add(metadataManager.getOzoneKey(volumeName, bucketName, args.getKeyName()));
+ }
+ }
+ Collections.sort(dbKeys);
+ String lastScannedDbKey = dbKeys.get(2); // The 3rd key
+
+ // create Lifecycle configuration
+ ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
+ ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
+ if (createPrefix) {
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true);
+ } else {
+ OmLCFilter.Builder filter = getOmLCFilterBuilder(rulePrefix, null, null);
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true);
+ }
+
+ // inject the resume state
+ String bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
+
+ OmLifecycleConfiguration policy = metadataManager.getLifecycleConfiguration(volumeName, bucketName);
+ OmLifecycleScanState.Builder stateBuilder = new OmLifecycleScanState.Builder()
+ .setBucketKey(bucketKey)
+ .setBucketObjID(bucketId)
+ .setLifecycleConfigurationUpdateID(policy.getUpdateID())
+ .setScanStartTime(System.currentTimeMillis());
+
+ if (bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
+ stateBuilder.setLastScannedKey(lastScannedDbKey);
+ stateBuilder.setLastScannedDir("");
+ stateBuilder.setLastScannedDirKey("");
+ } else {
+ stateBuilder.setLastScannedKey(lastScannedDbKey);
+ }
+ OmLifecycleScanState scanState = stateBuilder.build();
+ metadataManager.getLifecycleScanStateTable().put(bucketKey, scanState);
+ metadataManager.getLifecycleScanStateTable().addCacheEntry(new CacheKey<>(bucketKey),
+ CacheValue.get(1L, scanState));
+ OmLifecycleScanState state = metadataManager.getLifecycleScanStateTable().get(bucketKey);
+ assertNotNull(state);
+
+ // resume the service
+ keyLifecycleService.resume();
+
+ // wait for it to process
+ // it should skip the first 3 keys (index 0, 1, 2) since we set lastScannedDbKey as index 2.
+ // So it deletes only the last 2 keys (index 3 and 4).
+ int expectedDeleted = 2;
+ GenericTestUtils.waitFor(() ->
+ (getDeletedKeyCount() - initialDeletedKeyCount) >= expectedDeleted, WAIT_CHECK_INTERVAL, 10000);
+
+ // confirm it hasn't deleted all keys
+ assertEquals(testKeyCount - expectedDeleted, getKeyCount(bucketLayout) - initialKeyCount);
+ deleteLifecyclePolicy(volumeName, bucketName);
+ }
+
+ @ParameterizedTest
+ @MethodSource("parameters1")
+ void testBucketScanWithScanEndTime(BucketLayout bucketLayout, boolean createPrefix) throws IOException,
+ TimeoutException, InterruptedException {
+ final String volumeName = getTestName();
+ final String bucketName = uniqueObjectName("bucket");
+ String keyPrefix = "key";
+ String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : "key";
+ long initialDeletedKeyCount = getDeletedKeyCount();
+ long initialKeyCount = getKeyCount(bucketLayout);
+ int testKeyCount = 5;
+
+ // Suspend service so it doesn't process immediately after we create the policy
+ keyLifecycleService.suspend();
+
+ // create keys
+ List<OmKeyArgs> keyList =
+ createKeys(volumeName, bucketName, bucketLayout, testKeyCount, 1, keyPrefix, null);
+ assertEquals(testKeyCount, keyList.size());
+ GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == testKeyCount,
+ WAIT_CHECK_INTERVAL, 1000);
+
+ // determine db keys
+ List<String> dbKeys = new ArrayList<>();
+ long bucketId =
+ metadataManager.getBucketTable().get(metadataManager.getBucketKey(volumeName, bucketName)).getObjectID();
+ long volumeId = metadataManager.getVolumeTable().get(metadataManager.getVolumeKey(volumeName)).getObjectID();
+
+ for (OmKeyArgs args : keyList) {
+ if (bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
+ dbKeys.add(metadataManager.getOzonePathKey(volumeId, bucketId, bucketId, args.getKeyName()));
+ } else {
+ dbKeys.add(metadataManager.getOzoneKey(volumeName, bucketName, args.getKeyName()));
+ }
+ }
+ Collections.sort(dbKeys);
+ String lastScannedDbKey = dbKeys.get(2); // The 3rd key
+
+ // create Lifecycle configuration
+ ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
+ ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
+ if (createPrefix) {
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true);
+ } else {
+ OmLCFilter.Builder filter = getOmLCFilterBuilder(rulePrefix, null, null);
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true);
+ }
+
+ // inject the resume state but with ScanEndTime set!
+ String bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
+ OmLifecycleConfiguration policy = metadataManager.getLifecycleConfiguration(volumeName, bucketName);
+ OmLifecycleScanState.Builder stateBuilder = new OmLifecycleScanState.Builder()
+ .setBucketKey(bucketKey)
+ .setBucketObjID(bucketId)
+ .setLifecycleConfigurationUpdateID(policy.getUpdateID())
+ .setScanStartTime(System.currentTimeMillis())
+ .setScanEndTime(System.currentTimeMillis()); // Scan finished previously
+
+ if (bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
+ stateBuilder.setLastScannedKey(lastScannedDbKey);
+ stateBuilder.setLastScannedDir("");
+ } else {
+ stateBuilder.setLastScannedKey(lastScannedDbKey);
+ }
+ OmLifecycleScanState scanState = stateBuilder.build();
+ metadataManager.getLifecycleScanStateTable().put(bucketKey, scanState);
+ metadataManager.getLifecycleScanStateTable().addCacheEntry(new CacheKey<>(bucketKey),
+ CacheValue.get(1L, scanState));
+
+ // resume the service
+ keyLifecycleService.resume();
+
+ // wait for it to process
+ // Since ScanEndTime is set, it will ignore the lastScannedKey and start from the beginning!
+ // So it should delete ALL 5 keys.
+ int expectedDeleted = 5;
+ GenericTestUtils.waitFor(() ->
+ (getDeletedKeyCount() - initialDeletedKeyCount) >= expectedDeleted, WAIT_CHECK_INTERVAL, 10000);
+
+ // confirm it has deleted all keys
+ assertEquals(testKeyCount - expectedDeleted, getKeyCount(bucketLayout) - initialKeyCount);
+
+ deleteLifecyclePolicy(volumeName, bucketName);
+ }
+
+ @ParameterizedTest
+ @MethodSource("parameters1")
+ void testScanEmptyBucket(BucketLayout bucketLayout, boolean moveToTrash) throws Exception {
+ final String volumeName = getTestName();
+ final String bucketName = uniqueObjectName("bucket");
+
+ keyLifecycleService.setMoveToTrashEnabled(moveToTrash);
+
+ // Create empty bucket
+ createVolumeAndBucket(volumeName, bucketName, bucketLayout,
+ UserGroupInformation.getCurrentUser().getShortUserName());
+
+ // create Lifecycle configuration
+ ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
+ ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
+ OmLCFilter.Builder filter = bucketLayout == FILE_SYSTEM_OPTIMIZED ? getOmLCFilterBuilder("", null, null) :
+ getOmLCFilterBuilder("key", null, null);
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true);
+
+ // Wait until scan completes. Since the bucket is empty, it will scan immediately.
+ // We check if the scanEndTime is set.
+ String bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
+
+ GenericTestUtils.waitFor(() -> {
+ try {
+ OmLifecycleScanState scanState = metadataManager.getLifecycleScanStateTable().get(bucketKey);
+ return scanState != null && scanState.getScanEndTime() != null;
+ } catch (IOException e) {
+ return false;
+ }
+ }, WAIT_CHECK_INTERVAL, 10000);
+
+ deleteLifecyclePolicy(volumeName, bucketName);
+ }
+
+ @ParameterizedTest
+ @MethodSource("parameters1")
+ void testScanStateFailureDoesNotImpactScan(BucketLayout bucketLayout, boolean createPrefix) throws Exception {
+ final String volumeName = getTestName();
+ final String bucketName = uniqueObjectName("bucket");
+ String keyPrefix = "key";
+ String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : "key";
+ long initialDeletedKeyCount = getDeletedKeyCount();
+ long initialKeyCount = getKeyCount(bucketLayout);
+ // create keys
+ List<OmKeyArgs> keyList =
+ createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, keyPrefix, null);
+ assertEquals(KEY_COUNT, keyList.size());
+ GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == KEY_COUNT,
+ WAIT_CHECK_INTERVAL, 1000);
+
+ // Inject failure to LifecycleScanStateTable
+ Field tableField = OmMetadataManagerImpl.class.getDeclaredField("lifecycleScanStateTable");
+ tableField.setAccessible(true);
+ Table<String, OmLifecycleScanState> originalTable =
+ (Table<String, OmLifecycleScanState>) tableField.get(metadataManager);
+ Table<String, OmLifecycleScanState> spyTable = spy(originalTable);
+ doThrow(new RocksDatabaseException("Injected exception for testing")).when(spyTable).get(any());
+ doThrow(new RocksDatabaseException("Injected exception for testing")).when(spyTable).put(any(), any());
+ tableField.set(metadataManager, spyTable);
+
+ try {
+ // create Lifecycle configuration
+ ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
+ ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
+ if (createPrefix) {
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true);
+ } else {
+ OmLCFilter.Builder filter = getOmLCFilterBuilder(rulePrefix, null, null);
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true);
+ }
+
+ // Even though reading scan state fails, the deletion should still proceed normally.
+ GenericTestUtils.waitFor(() ->
+ (getDeletedKeyCount() - initialDeletedKeyCount) == KEY_COUNT, WAIT_CHECK_INTERVAL, 10000);
+ assertEquals(0, getKeyCount(bucketLayout) - initialKeyCount);
+
+ deleteLifecyclePolicy(volumeName, bucketName);
+ } finally {
+ // Restore original table
+ tableField.set(metadataManager, originalTable);
+ }
+ }
+
+ public Stream<Arguments> parameters12() {
+ return Stream.of(
+ arguments(FILE_SYSTEM_OPTIMIZED, 2),
+ arguments(FILE_SYSTEM_OPTIMIZED, 3),
+ arguments(FILE_SYSTEM_OPTIMIZED, 7),
+ arguments(BucketLayout.OBJECT_STORE, 2),
+ arguments(BucketLayout.OBJECT_STORE, 3),
+ arguments(BucketLayout.OBJECT_STORE, 7)
+ );
+ }
+
+ @ParameterizedTest
+ @MethodSource("parameters12")
+ void testNestedFSODirectoryScanResume(BucketLayout bucketLayout, int maxSize) throws Exception {
+ final String volumeName = getTestName();
+ final String bucketName = uniqueObjectName("bucket");
+ String prefix = "";
+ long initialDeletedKeyCount = getDeletedKeyCount();
+ long initialKeyCount = getKeyCount(bucketLayout);
+ long keyIterated = metrics.getNumKeyIterated().value();
+ int testKeyCount = 8;
+
+ keyLifecycleService.setListMaxSize(maxSize);
+ // Suspend service so it doesn't process immediately after we create the policy
+ keyLifecycleService.suspend();
+
+ createVolumeAndBucket(volumeName, bucketName, bucketLayout,
+ UserGroupInformation.getCurrentUser().getShortUserName());
+ /**
+ * Create nested directory and 8 keys inside
+ * /
+ * dir1 dir2 dir3 dir30
+ * / \ / \
+ * dir4 dir5 dir6 dir7
+ * / \
+ * dir8 dir9
+ *
+ * MaxSize = 2, lastScannedDir is dir9, lastScannedKey is key8
+ * MaxSize = 3, lastScannedDir is dir8, lastScannedKey is key6
+ * MaxSize = 7, lastScannedDir is dir5, lastScannedKey is key3
+ */
+ List<OmKeyArgs> keyList = new ArrayList<>();
+ keyList.add(createAndCommitKey(volumeName, bucketName, "dir1/dir4/key0", 1, null));
+ keyList.add(createAndCommitKey(volumeName, bucketName, "dir1/dir4/key1", 1, null));
+ keyList.add(createAndCommitKey(volumeName, bucketName, "dir1/dir5/key2", 1, null));
+ keyList.add(createAndCommitKey(volumeName, bucketName, "dir1/dir5/key3", 1, null));
+ keyList.add(createAndCommitKey(volumeName, bucketName, "dir3/dir6/dir8/key5", 1, null));
+ keyList.add(createAndCommitKey(volumeName, bucketName, "dir3/dir6/dir8/key6", 1, null));
+ keyList.add(createAndCommitKey(volumeName, bucketName, "dir3/dir6/dir9/key7", 1, null));
+ keyList.add(createAndCommitKey(volumeName, bucketName, "dir3/dir6/dir9/key8", 1, null));
+ if (bucketLayout == FILE_SYSTEM_OPTIMIZED) {
+ createDirectory(volumeName, bucketName, "dir2");
+ createDirectory(volumeName, bucketName, "dir3/dir7");
+ createDirectory(volumeName, bucketName, "dir30");
+ }
+
+ assertEquals(testKeyCount, keyList.size());
+ GenericTestUtils.waitFor(
+ () -> getKeyCount(bucketLayout) - initialKeyCount == testKeyCount, WAIT_CHECK_INTERVAL, 1000);
+
+ // Create Lifecycle configuration
+ ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
+ ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, prefix, null, date.toString(), true);
+
+ // Inject to cause resume case for FSO with nested directory
+ FaultInjectorImpl lastFaultInjector = new FaultInjectorImpl();
+ lastFaultInjector.setException(new IOException("Injected exception for testing"));
+ KeyLifecycleService.setInjectors(
+ Arrays.asList(new FaultInjectorImpl(), new FaultInjectorImpl(), lastFaultInjector));
+ // Resume the service
+ keyLifecycleService.resume();
+ KeyLifecycleService.getInjector(0).resume();
+ KeyLifecycleService.getInjector(1).resume();
+
+ // wait for scanState to be updated
+ String bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
+ GenericTestUtils.waitFor(() -> {
+ try {
+ OmLifecycleScanState scanState = metadataManager.getLifecycleScanStateTable().get(bucketKey);
+ if (bucketLayout == FILE_SYSTEM_OPTIMIZED) {
+ return scanState != null && scanState.getLastScannedDir() != null && scanState.getLastScannedKey() != null;
+ } else {
+ return scanState != null && scanState.getLastScannedKey() != null;
+ }
+ } catch (IOException e) {
+ return false;
+ }
+ }, WAIT_CHECK_INTERVAL, 10000);
+
+ OmLifecycleScanState scanState = metadataManager.getLifecycleScanStateTable().get(bucketKey);
+ if (stateSaveInternal != -1) {
+ if (maxSize == 2) {
+ if (bucketLayout == FILE_SYSTEM_OPTIMIZED) {
+ assertEquals("dir3/dir6/dir9", scanState.getLastScannedDir());
+ assertTrue(scanState.getLastScannedKey().endsWith("key8"));
+ } else {
+ assertTrue(scanState.getLastScannedKey().endsWith("key1"));
+ }
+ } else if (maxSize == 3) {
+ if (bucketLayout == FILE_SYSTEM_OPTIMIZED) {
+ assertEquals("dir3/dir6/dir8", scanState.getLastScannedDir());
+ assertTrue(scanState.getLastScannedKey().endsWith("key6"));
+ } else {
+ assertTrue(scanState.getLastScannedKey().endsWith("key2"));
+ }
+ } else {
+ if (bucketLayout == FILE_SYSTEM_OPTIMIZED) {
+ assertEquals("dir1/dir5", scanState.getLastScannedDir());
+ assertTrue(scanState.getLastScannedKey().endsWith("key3"));
+ } else {
+ assertTrue(scanState.getLastScannedKey().endsWith("key7"));
+ }
+ }
+ }
+
+ GenericTestUtils.waitFor(() ->
+ (getDeletedKeyCount() - initialDeletedKeyCount) == testKeyCount, WAIT_CHECK_INTERVAL, 10000);
+ // Confirm all keys are deleted
+ assertEquals(0, getKeyCount(bucketLayout) - initialKeyCount);
+ // Confirm iterated key number
+ GenericTestUtils.waitFor(() ->
+ testKeyCount == metrics.getNumKeyIterated().value() - keyIterated, WAIT_CHECK_INTERVAL, 5000);
+
deleteLifecyclePolicy(volumeName, bucketName);
}
@@ -283,12 +815,12 @@ void testOneKeyExpired(BucketLayout bucketLayout, boolean createPrefix) throws I
TimeoutException, InterruptedException {
final String volumeName = getTestName();
final String bucketName = uniqueObjectName("bucket");
- String prefix = "key";
+ String keyPrefix = "key";
long initialDeletedKeyCount = getDeletedKeyCount();
long initialKeyCount = getKeyCount(bucketLayout);
// create keys
List<OmKeyArgs> keyList =
- createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, prefix, null);
+ createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, keyPrefix, null);
// check there are keys in keyTable
assertEquals(KEY_COUNT, keyList.size());
GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == KEY_COUNT,
@@ -297,16 +829,266 @@ void testOneKeyExpired(BucketLayout bucketLayout, boolean createPrefix) throws I
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
int keyIndex = ThreadLocalRandom.current().nextInt(KEY_COUNT - 1);
+ String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : keyList.get(keyIndex).getKeyName();
if (createPrefix) {
- createLifecyclePolicy(volumeName, bucketName, bucketLayout,
- keyList.get(keyIndex).getKeyName(), null, date.toString(), true);
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true);
} else {
- OmLCFilter.Builder filter = getOmLCFilterBuilder(keyList.get(keyIndex).getKeyName(), null, null);
+ OmLCFilter.Builder filter = getOmLCFilterBuilder(rulePrefix, null, null);
createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true);
}
- GenericTestUtils.waitFor(() -> (getDeletedKeyCount() - initialDeletedKeyCount) == 1, WAIT_CHECK_INTERVAL, 10000);
- assertEquals(KEY_COUNT - 1, getKeyCount(bucketLayout) - initialKeyCount);
+ int expectedDeleteCount = bucketLayout == FILE_SYSTEM_OPTIMIZED ? KEY_COUNT : 1;
+ GenericTestUtils.waitFor(() -> (getDeletedKeyCount() - initialDeletedKeyCount) == expectedDeleteCount,
+ WAIT_CHECK_INTERVAL, 5000);
+ assertEquals(KEY_COUNT - expectedDeleteCount, getKeyCount(bucketLayout) - initialKeyCount);
+ deleteLifecyclePolicy(volumeName, bucketName);
+ }
+
+ public Stream<Arguments> parameters13() {
+ return Stream.of(
+ arguments("dirC", null, 3),
+ arguments("dirC/dir3", null, 3),
+ arguments("dirB", new String[]{"dirC"}, 2),
+ arguments("dirB/dir2", new String[]{"dirC"}, 2),
+ arguments("dirA", new String[]{"dirC", "dirB"}, 1),
+ arguments("dirA/dir1", new String[]{"dirC", "dirB"}, 1)
+ );
+ }
+
+ @ParameterizedTest
+ @MethodSource("parameters13")
+ void testDirectorySkippedAfterResume(String lastScannedDir, String[] skippedDir, int expectedDeleted)
+ throws Exception {
+ final String volumeName = getTestName();
+ final String bucketName = uniqueObjectName("bucket");
+ String prefix = "";
+ long initialDeletedKeyCount = getDeletedKeyCount();
+ long initialKeyCount = getKeyCount(BucketLayout.FILE_SYSTEM_OPTIMIZED);
+ keyLifecycleService.setListMaxSize(1);
+ // Suspend service so it doesn't process immediately after we create the policy
+ keyLifecycleService.suspend();
+
+ createVolumeAndBucket(volumeName, bucketName, BucketLayout.FILE_SYSTEM_OPTIMIZED,
+ UserGroupInformation.getCurrentUser().getShortUserName());
+
+ // Create 3 directories: dirA/dir1, dirB/dir2, dirC/dir3
+ // Inside each directory, create 1 keys.
+ int testKeyCount = 3;
+ List<OmKeyArgs> keyList = new ArrayList<>();
+
+ int i = 0;
+ for (String dir : Arrays.asList("dirA/dir1", "dirB/dir2", "dirC/dir3")) {
+ String keyName = dir + "/key" + i++;
+ keyList.add(createAndCommitKey(volumeName, bucketName, keyName, 1, null));
+ }
+
+ assertEquals(testKeyCount, keyList.size());
+ GenericTestUtils.waitFor(() -> getKeyCount(BucketLayout.FILE_SYSTEM_OPTIMIZED) - initialKeyCount == testKeyCount,
+ WAIT_CHECK_INTERVAL, 1000);
+
+ // Determine DB keys for the files
+ long bucketId =
+ metadataManager.getBucketTable().get(metadataManager.getBucketKey(volumeName, bucketName)).getObjectID();
+ long volumeId = metadataManager.getVolumeTable().get(metadataManager.getVolumeKey(volumeName)).getObjectID();
+
+ // Find dirB/dir2's objectID and its table key
+ String dirBKey = metadataManager.getOzonePathKey(volumeId, bucketId, bucketId, lastScannedDir);
+
+ // Create Lifecycle configuration
+ ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
+ ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
+ createLifecyclePolicy(volumeName, bucketName, BucketLayout.FILE_SYSTEM_OPTIMIZED, prefix,
+ null, date.toString(), true);
+
+ // Inject the resume state for FSO where lastScannedDir is dirB
+ String bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
+ OmLifecycleConfiguration policy = metadataManager.getLifecycleConfiguration(volumeName, bucketName);
+ OmLifecycleScanState.Builder stateBuilder = new OmLifecycleScanState.Builder()
+ .setBucketKey(bucketKey)
+ .setBucketObjID(bucketId)
+ .setLifecycleConfigurationUpdateID(policy.getUpdateID())
+ .setScanStartTime(System.currentTimeMillis())
+ .setLastScannedDir(lastScannedDir)
+ .setLastScannedDirKey(dirBKey);
+
+ OmLifecycleScanState scanState = stateBuilder.build();
+ metadataManager.getLifecycleScanStateTable().put(bucketKey, scanState);
+ metadataManager.getLifecycleScanStateTable().addCacheEntry(new CacheKey<>(bucketKey),
+ CacheValue.get(1L, scanState));
+
+ GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer.captureLogs(
+ LoggerFactory.getLogger(KeyLifecycleService.class));
+ // Resume the service
+ keyLifecycleService.resume();
+
+ // Wait for it to process
+ GenericTestUtils.waitFor(() ->
+ (getDeletedKeyCount() - initialDeletedKeyCount) == expectedDeleted, WAIT_CHECK_INTERVAL, 10000);
+
+ // Confirm it hasn't deleted dirA's keys
+ assertEquals(testKeyCount - expectedDeleted, getKeyCount(BucketLayout.FILE_SYSTEM_OPTIMIZED) - initialKeyCount);
+ if (skippedDir != null) {
+ Arrays.stream(skippedDir).forEach(
+ d -> assertTrue(logCapturer.getOutput().contains("Skip " + d)));
+ logCapturer.clearOutput();
+ }
+
+ deleteLifecyclePolicy(volumeName, bucketName);
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testLastScannedKeySeek(boolean keyBelongToDir) throws Exception {
+ final String volumeName = getTestName();
+ final String bucketName = uniqueObjectName("bucket");
+ String prefix = "";
+ long initialDeletedKeyCount = getDeletedKeyCount();
+ long initialKeyCount = getKeyCount(BucketLayout.FILE_SYSTEM_OPTIMIZED);
+ long keyIterated = metrics.getNumKeyIterated().value();
+
+ keyLifecycleService.suspend();
+
+ createVolumeAndBucket(volumeName, bucketName, BucketLayout.FILE_SYSTEM_OPTIMIZED,
+ UserGroupInformation.getCurrentUser().getShortUserName());
+
+ List<OmKeyArgs> keyList = new ArrayList<>();
+ keyList.add(createAndCommitKey(volumeName, bucketName, "dir1/key1", 1, null));
+ keyList.add(createAndCommitKey(volumeName, bucketName, "dir1/key2", 1, null));
+ keyList.add(createAndCommitKey(volumeName, bucketName, "dir2/key3", 1, null));
+ keyList.add(createAndCommitKey(volumeName, bucketName, "dir2/key4", 1, null));
+
+ assertEquals(4, keyList.size());
+ GenericTestUtils.waitFor(() -> getKeyCount(BucketLayout.FILE_SYSTEM_OPTIMIZED) - initialKeyCount == 4,
+ WAIT_CHECK_INTERVAL, 1000);
+
+ long bucketId =
+ metadataManager.getBucketTable().get(metadataManager.getBucketKey(volumeName, bucketName)).getObjectID();
+ long volumeId = metadataManager.getVolumeTable().get(metadataManager.getVolumeKey(volumeName)).getObjectID();
+
+ String dir1Key = metadataManager.getOzonePathKey(volumeId, bucketId, bucketId, "dir1");
+ long dir1Id = metadataManager.getDirectoryTable().get(
+ metadataManager.getOzonePathKey(volumeId, bucketId, bucketId, "dir1")).getObjectID();
+ long dir2Id = metadataManager.getDirectoryTable().get(
+ metadataManager.getOzonePathKey(volumeId, bucketId, bucketId, "dir2")).getObjectID();
+
+ // key2 is under dir1
+ String key2DbKey = metadataManager.getOzonePathKey(volumeId, bucketId, dir1Id, "key2");
+ // key3 is under dir2
+ String key3DbKey = metadataManager.getOzonePathKey(volumeId, bucketId, dir2Id, "key3");
+
+ ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
+ ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
+ OmLCFilter.Builder filter = getOmLCFilterBuilder(prefix, null, null);
+ createLifecyclePolicy(volumeName, bucketName, BucketLayout.FILE_SYSTEM_OPTIMIZED, null,
+ filter.build(), date.toString(), true);
+
+ // Set lastScannedDir to dir1, but lastScannedKey to key3 (which is in dir2)
+ String bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
+ OmLifecycleConfiguration policy = metadataManager.getLifecycleConfiguration(volumeName, bucketName);
+ OmLifecycleScanState.Builder stateBuilder = new OmLifecycleScanState.Builder()
+ .setBucketKey(bucketKey)
+ .setBucketObjID(bucketId)
+ .setLifecycleConfigurationUpdateID(policy.getUpdateID())
+ .setScanStartTime(System.currentTimeMillis());
+
+ if (keyBelongToDir) {
+ stateBuilder.setLastScannedDir("dir1").setLastScannedDirKey(dir1Key).setLastScannedKey(key2DbKey);
+ } else {
+ stateBuilder.setLastScannedDir("dir1").setLastScannedDirKey(dir1Key).setLastScannedKey(key3DbKey);
+ }
+
+ OmLifecycleScanState scanState = stateBuilder.build();
+ metadataManager.getLifecycleScanStateTable().put(bucketKey, scanState);
+ metadataManager.getLifecycleScanStateTable().addCacheEntry(new CacheKey<>(bucketKey),
+ CacheValue.get(1L, scanState));
+
+ GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer.captureLogs(
+ LoggerFactory.getLogger(KeyLifecycleService.class));
+ keyLifecycleService.resume();
+
+ // dir1 can be fully evaluated depending on whether lastScannedKey belong to it (no seek) or not
+ // dir2 should be skipped dir2 > dir1
+ int expectedDeleted = 2;
+ GenericTestUtils.waitFor(() ->
+ (getDeletedKeyCount() - initialDeletedKeyCount) >= expectedDeleted, WAIT_CHECK_INTERVAL, 5000);
+ assertEquals(keyList.size() - expectedDeleted, getKeyCount(BucketLayout.FILE_SYSTEM_OPTIMIZED) - initialKeyCount);
+ GenericTestUtils.waitFor(() ->
+ expectedDeleted == metrics.getNumKeyIterated().value() - keyIterated, WAIT_CHECK_INTERVAL, 5000);
+ if (keyBelongToDir) {
+ assertTrue(logCapturer.getOutput().contains("Seek to key"));
+ } else {
+ assertFalse(logCapturer.getOutput().contains("Seek to key"));
+ }
+ deleteLifecyclePolicy(volumeName, bucketName);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"FILE_SYSTEM_OPTIMIZED", "OBJECT_STORE"})
+ void testBucketRootScannedDirResume(BucketLayout layout) throws Exception {
+ final String volumeName = getTestName();
+ final String bucketName = uniqueObjectName("bucket");
+ String prefix = "";
+ long initialDeletedKeyCount = getDeletedKeyCount();
+ long initialKeyCount = getKeyCount(layout);
+ long keyIterated = metrics.getNumKeyIterated().value();
+
+ keyLifecycleService.suspend();
+
+ createVolumeAndBucket(volumeName, bucketName, layout,
+ UserGroupInformation.getCurrentUser().getShortUserName());
+
+ List<OmKeyArgs> keyList = new ArrayList<>();
+ keyList.add(createAndCommitKey(volumeName, bucketName, "key1", 1, null));
+ keyList.add(createAndCommitKey(volumeName, bucketName, "key2", 1, null));
+ keyList.add(createAndCommitKey(volumeName, bucketName, "key3", 1, null));
+
+ assertEquals(3, keyList.size());
+ GenericTestUtils.waitFor(() -> getKeyCount(layout) - initialKeyCount == 3,
+ WAIT_CHECK_INTERVAL, 1000);
+
+ long bucketId =
+ metadataManager.getBucketTable().get(metadataManager.getBucketKey(volumeName, bucketName)).getObjectID();
+ long volumeId = metadataManager.getVolumeTable().get(metadataManager.getVolumeKey(volumeName)).getObjectID();
+
+ // key2 in bucket root
+ String key2DbKey = layout == FILE_SYSTEM_OPTIMIZED ?
+ metadataManager.getOzonePathKey(volumeId, bucketId, bucketId, "key2") :
+ metadataManager.getOzoneKey(volumeName, bucketName, "key2");
+
+ ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
+ ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
+ OmLCFilter.Builder filter = getOmLCFilterBuilder(prefix, null, null);
+ createLifecyclePolicy(volumeName, bucketName, layout, null,
+ filter.build(), date.toString(), true);
+
+ // Set lastScannedDir to "" (bucket root) and lastScannedKey to key2
+ String bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
+ OmLifecycleConfiguration policy = metadataManager.getLifecycleConfiguration(volumeName, bucketName);
+ OmLifecycleScanState.Builder stateBuilder = new OmLifecycleScanState.Builder()
+ .setBucketKey(bucketKey)
+ .setBucketObjID(bucketId)
+ .setLifecycleConfigurationUpdateID(policy.getUpdateID())
+ .setScanStartTime(System.currentTimeMillis())
+ .setLastScannedDir("")
+ .setLastScannedDirKey("")
+ .setLastScannedKey(key2DbKey);
+
+ OmLifecycleScanState scanState = stateBuilder.build();
+ metadataManager.getLifecycleScanStateTable().put(bucketKey, scanState);
+ metadataManager.getLifecycleScanStateTable().addCacheEntry(new CacheKey<>(bucketKey),
+ CacheValue.get(1L, scanState));
+
+ keyLifecycleService.resume();
+
+ // It should seek to key2. key1 is skipped, key2 is skipped too.
+ // So key1 and key2 are skipped, key3 is deleted.
+ int expectedDeleted = 1;
+ GenericTestUtils.waitFor(() ->
+ (getDeletedKeyCount() - initialDeletedKeyCount) == expectedDeleted, WAIT_CHECK_INTERVAL, 10000);
+
+ assertEquals(2, getKeyCount(layout) - initialKeyCount);
+ GenericTestUtils.waitFor(() ->
+ expectedDeleted == metrics.getNumKeyIterated().value() - keyIterated, WAIT_CHECK_INTERVAL, 5000);
deleteLifecyclePolicy(volumeName, bucketName);
}
@@ -328,11 +1110,11 @@ void testOnlyKeyExpired(BucketLayout bucketLayout, boolean createPrefix) throws
// create Lifecycle configuration
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
+ String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : keyArg.getKeyName();
if (createPrefix) {
- createLifecyclePolicy(volumeName, bucketName, bucketLayout,
- keyArg.getKeyName(), null, date.toString(), true);
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true);
} else {
- OmLCFilter.Builder filter = getOmLCFilterBuilder(keyArg.getKeyName(), null, null);
+ OmLCFilter.Builder filter = getOmLCFilterBuilder(rulePrefix, null, null);
createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true);
}
@@ -414,13 +1196,13 @@ void testAllKeyExpiredWithAndOperator(BucketLayout bucketLayout) throws IOExcept
TimeoutException, InterruptedException {
final String volumeName = getTestName();
final String bucketName = uniqueObjectName("bucket");
- String prefix = "key";
+ String keyPrefix = "key";
long initialDeletedKeyCount = getDeletedKeyCount();
long initialKeyCount = getKeyCount(bucketLayout);
Map<String, String> tags = ImmutableMap.of("app", "spark", "user", "ozone");
// create keys
List<OmKeyArgs> keyList =
- createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, prefix, tags);
+ createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, keyPrefix, tags);
// check there are keys in keyTable
assertEquals(KEY_COUNT, keyList.size());
GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == KEY_COUNT,
@@ -428,7 +1210,8 @@ void testAllKeyExpiredWithAndOperator(BucketLayout bucketLayout) throws IOExcept
// create Lifecycle configuration
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
- OmLifecycleRuleAndOperator andOperator = getOmLCAndOperatorBuilder(prefix, tags).build();
+ String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : keyPrefix;
+ OmLifecycleRuleAndOperator andOperator = getOmLCAndOperatorBuilder(rulePrefix, tags).build();
OmLCFilter.Builder filter = getOmLCFilterBuilder(null, null, andOperator);
createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true);
@@ -445,15 +1228,15 @@ void testOneKeyExpiredWithAndOperator(BucketLayout bucketLayout) throws IOExcept
int keyCount = KEY_COUNT;
final String volumeName = getTestName();
final String bucketName = uniqueObjectName("bucket");
- String prefix = "key";
+ String keyPrefix = "key";
long initialDeletedKeyCount = getDeletedKeyCount();
long initialKeyCount = getKeyCount(bucketLayout);
Map<String, String> tags = ImmutableMap.of("app", "spark", "user", "ozone");
// create keys without tags
List<OmKeyArgs> keyList =
- createKeys(volumeName, bucketName, bucketLayout, keyCount, 1, prefix, null);
+ createKeys(volumeName, bucketName, bucketLayout, keyCount, 1, keyPrefix, null);
// create one more key with tag
- final String keyName = uniqueObjectName(prefix);
+ final String keyName = uniqueObjectName(keyPrefix);
// Create the key
OmKeyArgs keyArg = createAndCommitKey(volumeName, bucketName, keyName, 1, tags);
keyList.add(keyArg);
@@ -466,7 +1249,8 @@ void testOneKeyExpiredWithAndOperator(BucketLayout bucketLayout) throws IOExcept
// create Lifecycle configuration
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
- OmLifecycleRuleAndOperator andOperator = getOmLCAndOperatorBuilder(prefix, tags).build();
+ String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : keyPrefix;
+ OmLifecycleRuleAndOperator andOperator = getOmLCAndOperatorBuilder(rulePrefix, tags).build();
OmLCFilter.Builder filter = getOmLCFilterBuilder(null, null, andOperator);
createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true);
@@ -573,10 +1357,12 @@ void testRootSlashPrefix(BucketLayout bucketLayout, String prefix)
@MethodSource("parameters1")
void testSlashPrefix(BucketLayout bucketLayout, boolean createPrefix)
throws IOException, TimeoutException, InterruptedException {
- assumeTrue(bucketLayout != FILE_SYSTEM_OPTIMIZED);
+ // FSO bucket must end with "/". "/" is also invalid prefix for FSO.
+ assumeTrue(bucketLayout == OBJECT_STORE);
final String volumeName = getTestName();
final String bucketName = uniqueObjectName("bucket");
String keyPrefix = "key";
+ String rulePrefix = "/" + keyPrefix;
long initialDeletedKeyCount = getDeletedKeyCount();
long initialKeyCount = getKeyCount(bucketLayout);
// create keys
@@ -590,9 +1376,9 @@ void testSlashPrefix(BucketLayout bucketLayout, boolean createPrefix)
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
if (createPrefix) {
- createLifecyclePolicy(volumeName, bucketName, bucketLayout, "/" + keyPrefix, null, date.toString(), true);
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true);
} else {
- OmLCFilter.Builder filter = getOmLCFilterBuilder("/" + keyPrefix, null, null);
+ OmLCFilter.Builder filter = getOmLCFilterBuilder(rulePrefix, null, null);
createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true);
}
@@ -612,6 +1398,7 @@ void testSlashPrefix(BucketLayout bucketLayout, boolean createPrefix)
@MethodSource("parameters1")
void testSlashKey(BucketLayout bucketLayout, boolean createPrefix)
throws IOException, TimeoutException, InterruptedException {
+ // FSO bucket doesn't allow "//" in prefix.
assumeTrue(bucketLayout != FILE_SYSTEM_OPTIMIZED);
final String volumeName = getTestName();
final String bucketName = uniqueObjectName("bucket");
@@ -645,6 +1432,7 @@ void testSlashKey(BucketLayout bucketLayout, boolean createPrefix)
@ValueSource(strings = {"FILE_SYSTEM_OPTIMIZED", "OBJECT_STORE"})
void testSlashKeyWithAndOperator(BucketLayout bucketLayout)
throws IOException, TimeoutException, InterruptedException {
+ // FSO bucket doesn't allow "//" in prefix.
assumeTrue(bucketLayout != FILE_SYSTEM_OPTIMIZED);
final String volumeName = getTestName();
final String bucketName = uniqueObjectName("bucket");
@@ -676,6 +1464,7 @@ void testSlashKeyWithAndOperator(BucketLayout bucketLayout)
@ValueSource(strings = {"FILE_SYSTEM_OPTIMIZED", "OBJECT_STORE"})
void testSlashPrefixWithAndOperator(BucketLayout bucketLayout)
throws IOException, InterruptedException, TimeoutException {
+ // FSO bucket must end with "/". "/" is also invalid prefix for FSO.
assumeTrue(bucketLayout != FILE_SYSTEM_OPTIMIZED);
final String volumeName = getTestName();
final String bucketName = uniqueObjectName("bucket");
@@ -709,14 +1498,14 @@ void testComplexPrefix(BucketLayout bucketLayout) throws IOException,
TimeoutException, InterruptedException {
final String volumeName = getTestName();
final String bucketName = uniqueObjectName("bucket");
- String prefix = "dir1/dir2/dir3/key";
+ String keyPrefix = "dir1/dir2/dir3/key";
long initialDeletedKeyCount = getDeletedKeyCount();
long initialKeyCount = getKeyCount(bucketLayout);
long initialNumDeletedKey = metrics.getNumKeyDeleted().value();
long initialSizeDeletedKey = metrics.getSizeKeyDeleted().value();
// create keys
List<OmKeyArgs> keyList =
- createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, prefix, null);
+ createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, keyPrefix, null);
// check there are keys in keyTable
assertEquals(KEY_COUNT, keyList.size());
GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == KEY_COUNT,
@@ -724,7 +1513,8 @@ void testComplexPrefix(BucketLayout bucketLayout) throws IOException,
// create Lifecycle configuration
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
- createLifecyclePolicy(volumeName, bucketName, bucketLayout, prefix, null, date.toString(), true);
+ String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "dir1/dir2/dir3/" : keyPrefix;
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true);
GenericTestUtils.waitFor(() ->
(getDeletedKeyCount() - initialDeletedKeyCount) == KEY_COUNT, WAIT_CHECK_INTERVAL, 10000);
@@ -741,7 +1531,7 @@ void testPrefixNotMatch(BucketLayout bucketLayout) throws IOException, Interrupt
final String volumeName = getTestName();
final String bucketName = uniqueObjectName("bucket");
String keyPrefix = "dir1/dir2/dir3/key";
- String filterPrefix = "dir1/dir2/dir4/key";
+ String filterPrefix = "dir1/dir2/dir4/";
long initialDeletedKeyCount = getDeletedKeyCount();
long initialKeyCount = getKeyCount(bucketLayout);
// create keys
@@ -769,12 +1559,12 @@ void testExpireKeysUnderDirectory(BucketLayout bucketLayout) throws IOException,
TimeoutException, InterruptedException {
final String volumeName = getTestName();
final String bucketName = uniqueObjectName("bucket");
- String prefix = "dir1/dir2/dir3/key";
+ String keyPrefix = "dir1/dir2/dir3/key";
long initialDeletedKeyCount = getDeletedKeyCount();
long initialKeyCount = getKeyCount(bucketLayout);
// create keys
List<OmKeyArgs> keyList =
- createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, prefix, null);
+ createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, keyPrefix, null);
// check there are keys in keyTable
assertEquals(KEY_COUNT, keyList.size());
GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == KEY_COUNT,
@@ -788,7 +1578,8 @@ void testExpireKeysUnderDirectory(BucketLayout bucketLayout) throws IOException,
// create Lifecycle configuration
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
- createLifecyclePolicy(volumeName, bucketName, bucketLayout, prefix, null, date.toString(), true);
+ String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "dir1/dir2/dir3/" : keyPrefix;
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true);
GenericTestUtils.waitFor(() ->
(getDeletedKeyCount() - initialDeletedKeyCount) == KEY_COUNT, WAIT_CHECK_INTERVAL, 10000);
@@ -799,8 +1590,6 @@ void testExpireKeysUnderDirectory(BucketLayout bucketLayout) throws IOException,
public Stream<Arguments> parameters3() {
return Stream.of(
arguments("dir1/dir2/dir3/key", "dir1/dir2/dir3/", "dir1/dir2/dir3"),
- arguments("dir1/dir2/dir3/key", "dir1/dir2/dir3", "dir1/dir2/dir3"),
- arguments("dir1/key", "dir1", "dir1"),
arguments("dir1/key", "dir1/", "dir1"));
}
@@ -856,50 +1645,19 @@ void testMatchedDirectoryNotDeleted(String keyPrefix, String rulePrefix, String
public Stream<Arguments> parameters4() {
return Stream.of(
- arguments("dir1/dir2/dir3", "dir1/dir2/dir3", 3, 0, true, false),
- arguments("dir1/dir2/dir3", "dir1/dir2/dir3", 3, 0, false, true),
- arguments("/dir1/dir2/dir3", "dir1/dir2/dir3", 3, 1, true, false),
- arguments("/dir1/dir2/dir3", "dir1/dir2/dir3", 3, 1, false, true),
- arguments("/dir1/dir2/dir3/", "dir1/dir2/dir3/", 3, 0, true, false),
- arguments("/dir1/dir2/dir3/", "dir1/dir2/dir3/", 3, 0, false, true),
- arguments("/dir1//dir2//dir3//", "dir1/dir2/dir3/", 3, 0, true, false),
- arguments("/dir1//dir2//dir3//", "dir1/dir2/dir3/", 3, 0, false, true),
- arguments("/dir1//dir2/dir3/", "dir1/dir2/dir", 3, 1, true, false),
- arguments("/dir1//dir2/dir3/", "dir1/dir2/dir", 3, 1, false, true),
- arguments("/dir1//dir2/dir3/", "dir1/dir2/dir/", 3, 0, true, false),
- arguments("/dir1//dir2/dir3/", "dir1/dir2/dir/", 3, 0, false, true),
- arguments("dir1/dir2", "dir1/dir2/", 2, 0, true, false),
- arguments("dir1/dir2", "dir1/dir2/", 2, 0, false, true),
- arguments("/dir1/dir2", "dir1/dir2", 2, 1, true, false),
- arguments("/dir1/dir2", "dir1/dir2", 2, 1, false, true),
- arguments("/dir1/dir2/", "dir1/dir2", 2, 1, true, false),
- arguments("/dir1/dir2/", "dir1/dir2", 2, 1, false, true),
- arguments("/dir1//dir2//", "dir1/dir2/", 2, 0, true, false),
- arguments("/dir1//dir2//", "dir1/dir2/", 2, 0, false, true),
- arguments("/dir1//dir2//", "dir1/dir", 2, 1, true, false),
- arguments("/dir1//dir2//", "dir1/dir", 2, 1, false, true),
- arguments("/dir1//dir2//", "dir1/dir/", 2, 0, true, false),
- arguments("/dir1//dir2//", "dir1/dir/", 2, 0, false, true),
- arguments("dir1", "dir1/", 1, 0, true, false),
- arguments("dir1", "dir1/", 1, 0, false, true),
- arguments("/dir1", "dir1", 1, 1, true, false),
- arguments("/dir1", "dir1", 1, 1, false, true),
- arguments("/dir1/", "dir1", 1, 1, true, false),
- arguments("/dir1/", "dir1", 1, 1, false, true),
- arguments("/dir1//", "dir1/", 1, 0, true, false),
- arguments("/dir1//", "dir1/", 1, 0, false, true),
- arguments("/dir1//", "dir", 1, 1, true, false),
- arguments("/dir1//", "dir", 1, 1, false, true),
- arguments("/dir1//", "dir/", 1, 0, true, false),
- arguments("/dir1//", "dir/", 1, 0, false, true)
+ arguments("dir1/dir2/dir3//", "dir1/dir2/dir3/", 3, true, false),
+ arguments("dir1/dir2/dir3//", "dir1/dir2/dir3/", 3, false, true),
+ arguments("dir1/dir2//", "dir1/dir2/", 2, true, false),
+ arguments("dir1/dir2//", "dir1/dir2/", 2, false, true),
+ arguments("dir1//", "dir1/", 1, true, false),
+ arguments("dir1//", "dir1/", 1, false, true)
);
}
@ParameterizedTest
@MethodSource("parameters4")
- void testExpireOnlyDirectory(String dirName, String prefix, int dirDepth, int deletedDirCount,
- boolean createPrefix, boolean createFilterPrefix) throws IOException,
- TimeoutException, InterruptedException {
+ void testPrefixDirectoryNotExpired(String dirName, String prefix, int dirDepth, boolean createPrefix,
+ boolean createFilterPrefix) throws IOException, TimeoutException, InterruptedException {
final String volumeName = getTestName();
final String bucketName = uniqueObjectName("bucket");
long initialDeletedDirCount = getDeletedDirectoryCount();
@@ -916,6 +1674,10 @@ void testExpireOnlyDirectory(String dirName, String prefix, int dirDepth, int de
assertEquals(dirDepth, getDirCount() - initialDirCount);
assertEquals(0, getDeletedDirectoryCount() - initialDeletedDirCount);
+ GenericTestUtils.LogCapturer log =
+ GenericTestUtils.LogCapturer.captureLogs(
+ LoggerFactory.getLogger(KeyLifecycleService.class));
+
// create Lifecycle configuration
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
@@ -928,13 +1690,171 @@ void testExpireOnlyDirectory(String dirName, String prefix, int dirDepth, int de
null, filter.build(), date.toString(), true);
}
- GenericTestUtils.waitFor(
- () -> (getDeletedDirectoryCount() - initialDeletedDirCount) == deletedDirCount, WAIT_CHECK_INTERVAL, 10000);
- assertEquals(dirDepth - deletedDirCount, getDirCount() - initialDirCount);
- assertEquals(deletedDirCount, metrics.getNumDirDeleted().value() - initialNumDeletedDir);
+ GenericTestUtils.waitFor(() -> log.getOutput().contains("Prefix directory " + prefix + " doesn't get expired"),
+ WAIT_CHECK_INTERVAL, 10000);
+ assertEquals(dirDepth, getDirCount() - initialDirCount);
+ assertEquals(0, metrics.getNumDirDeleted().value() - initialNumDeletedDir);
+ deleteLifecyclePolicy(volumeName, bucketName);
+ }
+
+ @Test
+ void testConsolidatedPrefixNotHappen() throws IOException, TimeoutException, InterruptedException {
+ final String volumeName = getTestName();
+ final String bucketName = uniqueObjectName("bucket");
+ long initialDeletedKeyCount = getDeletedKeyCount();
+ String dir1 = "dir/dir1/dir2/";
+ String dir2 = "log/log1/log2/";
+
+ // Create the directories
+ createVolumeAndBucket(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED,
+ UserGroupInformation.getCurrentUser().getShortUserName());
+ createDirectory(volumeName, bucketName, dir1);
+ createDirectory(volumeName, bucketName, dir2);
+ KeyInfoWithVolumeContext keyInfo = getDirectory(volumeName, bucketName, dir1);
+ assertFalse(keyInfo.getKeyInfo().isFile());
+ keyInfo = getDirectory(volumeName, bucketName, dir2);
+ assertFalse(keyInfo.getKeyInfo().isFile());
+ List<OmKeyArgs> keyList = new ArrayList<>();
+ keyList.add(createAndCommitKey(volumeName, bucketName, dir1 + "key1", 1, null));
+ keyList.add(createAndCommitKey(volumeName, bucketName, dir2 + "key2", 1, null));
+
+ Thread.sleep(SERVICE_INTERVAL);
+
+ // create Lifecycle configuration
+ ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
+ ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
+ List<OmLCRule> ruleList = new ArrayList<>();
+ String ruleID1 = String.valueOf(OBJECT_ID_COUNTER.getAndIncrement());
+ String ruleID2 = String.valueOf(OBJECT_ID_COUNTER.getAndIncrement());
+ ruleList.add(new OmLCRule.Builder().setId(ruleID1)
+ .setEnabled(true).setPrefix(dir1)
+ .setAction(new OmLCExpiration.Builder().setDate(date.toString()).build())
+ .build());
+ ruleList.add(new OmLCRule.Builder().setId(ruleID2)
+ .setEnabled(true).setPrefix(dir2)
+ .setAction(new OmLCExpiration.Builder().setDate(date.toString()).build())
+ .build());
+ createLifecyclePolicy(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED, ruleList);
+
+ GenericTestUtils.waitFor(() -> getDeletedKeyCount() - initialDeletedKeyCount == keyList.size(),
+ WAIT_CHECK_INTERVAL, 5000);
deleteLifecyclePolicy(volumeName, bucketName);
}
+ public Stream<Arguments> parameters41() {
+ return Stream.of(
+ arguments("dir/dir1/", "dir/dir1/dir2/", null, null, true,
+ "Prefix directory dir/dir1/dir2/ doesn't get expired", 1, "dir/dir1/", null),
+ arguments("dir/dir1/", "dir/dir2/", null, null, false,
+ "Prefix directory dir/dir2/ doesn't get expired", 2, "dir/dir2/", null),
+ arguments("dir1/dir2/", "log1/log2/", null, null, false,
+ "Prefix directory dir1/dir2/ doesn't get expired", 2, "log1/log2/", null),
+ arguments("dir/dir1/", "dir/dir1/dir2/", "dir/", null, true,
+ "Prefix directory dir/dir1/dir2/ doesn't get expired", 1, "dir/", null),
+ arguments("dir/dir1/dir2/", "dir/", "dir/dir1/", null, true,
+ "Prefix directory dir/dir1/dir2/ doesn't get expired", 1, "dir/", null),
+ arguments("dir/", "dir/dir1/", "dir/dir1/dir2/", null, true,
+ "Prefix directory dir/dir1/dir2/ doesn't get expired", 1, "dir/", null),
+ arguments("dir/dir1/", "log/log1/", "dir/dir1/dir2/", null, true,
+ "Prefix directory dir/dir1/dir2/ doesn't get expired", 2, "log/log1/", "dir/dir1/"),
+ arguments("dir/dir1/", "log/log1/", "data/data1/", "log/log1/log2/", true,
+ "Prefix directory data/data1/ doesn't get expired", 3, "log/log1/", "data/data1/")
+ );
+ }
+
+ @ParameterizedTest
+ @MethodSource("parameters41")
+ @SuppressWarnings("parameternumber")
+ void testConsolidatedPrefixDirectoryNotExpired(String dir1, String dir2, String dir3, String dir4,
+ boolean shouldConsolidateRule, String expectedLog, int consolidatedRuleListSize,
+ String firstConsolidatedPrefix, String lastConsolidatedPrefix)
+ throws IOException, TimeoutException, InterruptedException {
+ final String volumeName = getTestName();
+ final String bucketName = uniqueObjectName("bucket");
+ long initialDeletedKeyCount = getDeletedKeyCount();
+ long initialKeyIterated = metrics.getNumKeyIterated().value();
+ long initialKeyDeleted = metrics.getNumKeyDeleted().value();
+
+ // Create the directories
+ createVolumeAndBucket(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED,
+ UserGroupInformation.getCurrentUser().getShortUserName());
+ createDirectory(volumeName, bucketName, dir1);
+ createDirectory(volumeName, bucketName, dir2);
+ KeyInfoWithVolumeContext keyInfo = getDirectory(volumeName, bucketName, dir1);
+ assertFalse(keyInfo.getKeyInfo().isFile());
+ keyInfo = getDirectory(volumeName, bucketName, dir2);
+ assertFalse(keyInfo.getKeyInfo().isFile());
+ List<OmKeyArgs> keyList = new ArrayList<>();
+ keyList.add(createAndCommitKey(volumeName, bucketName, dir1 + "key1", 1, null));
+ keyList.add(createAndCommitKey(volumeName, bucketName, dir2 + "key2", 1, null));
+
+ Thread.sleep(SERVICE_INTERVAL);
+ KeyLifecycleService.setTest(true);
+ KeyLifecycleService.reSetConsolidatedRuleList();
+
+ GenericTestUtils.LogCapturer log =
+ GenericTestUtils.LogCapturer.captureLogs(
+ LoggerFactory.getLogger(KeyLifecycleService.class));
+
+ // create Lifecycle configuration
+ ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
+ ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
+ List<OmLCRule> ruleList = new ArrayList<>();
+ String ruleID1 = String.valueOf(OBJECT_ID_COUNTER.getAndIncrement());
+ String ruleID2 = String.valueOf(OBJECT_ID_COUNTER.getAndIncrement());
+ ruleList.add(new OmLCRule.Builder().setId(ruleID1)
+ .setEnabled(true).setPrefix(dir1)
+ .setAction(new OmLCExpiration.Builder().setDate(date.toString()).build())
+ .build());
+ ruleList.add(new OmLCRule.Builder().setId(ruleID2)
+ .setEnabled(true).setPrefix(dir2)
+ .setAction(new OmLCExpiration.Builder().setDate(date.toString()).build())
+ .build());
+ if (dir3 != null) {
+ String ruleID3 = String.valueOf(OBJECT_ID_COUNTER.getAndIncrement());
+ ruleList.add(new OmLCRule.Builder().setId(ruleID3)
+ .setEnabled(true).setPrefix(dir3)
+ .setAction(new OmLCExpiration.Builder().setDate(date.toString()).build())
+ .build());
+ keyList.add(createAndCommitKey(volumeName, bucketName, dir3 + "key3", 1, null));
+ }
+ if (dir4 != null) {
+ String ruleID4 = String.valueOf(OBJECT_ID_COUNTER.getAndIncrement());
+ ruleList.add(new OmLCRule.Builder().setId(ruleID4)
+ .setEnabled(true).setPrefix(dir4)
+ .setAction(new OmLCExpiration.Builder().setDate(date.toString()).build())
+ .build());
+ keyList.add(createAndCommitKey(volumeName, bucketName, dir4 + "key4", 1, null));
+ }
+ createLifecyclePolicy(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED, ruleList);
+
+ try {
+ if (shouldConsolidateRule) {
+ GenericTestUtils.waitFor(() -> log.getOutput().contains("Consolidate"), WAIT_CHECK_INTERVAL, 5000);
+ }
+ if (expectedLog != null) {
+ GenericTestUtils.waitFor(() -> log.getOutput().contains(expectedLog), WAIT_CHECK_INTERVAL, 5000);
+ }
+ GenericTestUtils.waitFor(() -> getDeletedKeyCount() - initialDeletedKeyCount == keyList.size(),
+ WAIT_CHECK_INTERVAL, 5000);
+ GenericTestUtils.waitFor(() -> keyList.size() == metrics.getNumKeyIterated().value() - initialKeyIterated,
+ WAIT_CHECK_INTERVAL, 5000);
+ GenericTestUtils.waitFor(() -> keyList.size() == metrics.getNumKeyDeleted().value() - initialKeyDeleted,
+ WAIT_CHECK_INTERVAL, 5000);
+ GenericTestUtils.waitFor(() -> {
+ List<KeyLifecycleService.RuleListWithDirectoryList> list = KeyLifecycleService.getConsolidatedRuleList();
+ boolean sizeMatch = list != null && list.size() == consolidatedRuleListSize;
+ boolean firstPrefixMatch = list != null &&
+ firstConsolidatedPrefix.equals(list.get(0).getConsolidatedPrefix());
+ boolean lastPrefixMatch = lastConsolidatedPrefix == null ? true :
+ list != null && lastConsolidatedPrefix.equals(list.get(list.size() - 1).getConsolidatedPrefix());
+ return sizeMatch && firstPrefixMatch && lastPrefixMatch;
+ }, WAIT_CHECK_INTERVAL, 5000);
+ } finally {
+ deleteLifecyclePolicy(volumeName, bucketName);
+ }
+ }
+
@ParameterizedTest
@ValueSource(strings = {"FILE_SYSTEM_OPTIMIZED", "OBJECT_STORE"})
void testExpireNonExistDirectory(BucketLayout bucketLayout)
@@ -962,7 +1882,7 @@ void testExpireNonExistDirectory(BucketLayout bucketLayout)
// create Lifecycle configuration
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
- createLifecyclePolicy(volumeName, bucketName, bucketLayout, "dir1/dir2/dir4", null, date.toString(), true);
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, "dir1/dir2/dir4/", null, date.toString(), true);
Thread.sleep(EXPIRE_SECONDS);
@@ -1007,12 +1927,12 @@ void testOneRuleDisabledOneRuleEnabled(BucketLayout bucketLayout)
throws IOException, InterruptedException, TimeoutException {
final String volumeName = getTestName();
final String bucketName = uniqueObjectName("bucket");
- String prefix = "key";
+ String keyPrefix = "key";
long initialDeletedKeyCount = getDeletedKeyCount();
long initialKeyCount = getKeyCount(bucketLayout);
// create keys
List<OmKeyArgs> keyList =
- createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, prefix, null);
+ createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, keyPrefix, null);
// check there are keys in keyTable
assertEquals(KEY_COUNT, keyList.size());
GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == KEY_COUNT,
@@ -1021,12 +1941,13 @@ void testOneRuleDisabledOneRuleEnabled(BucketLayout bucketLayout)
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
List<OmLCRule> ruleList = new ArrayList<>();
+ String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : keyPrefix;
ruleList.add(new OmLCRule.Builder().setId(String.valueOf(OBJECT_ID_COUNTER.getAndIncrement()))
- .setEnabled(false).setPrefix(prefix)
+ .setEnabled(false).setPrefix(rulePrefix)
.setAction(new OmLCExpiration.Builder().setDate(date.toString()).build())
.build());
ruleList.add(new OmLCRule.Builder().setId(String.valueOf(OBJECT_ID_COUNTER.getAndIncrement()))
- .setEnabled(true).setPrefix(prefix)
+ .setEnabled(true).setPrefix(rulePrefix)
.setAction(new OmLCExpiration.Builder().setDate(date.toString()).build())
.build());
createLifecyclePolicy(volumeName, bucketName, bucketLayout, ruleList);
@@ -1041,6 +1962,7 @@ void testOneRuleDisabledOneRuleEnabled(BucketLayout bucketLayout)
@ValueSource(strings = {"FILE_SYSTEM_OPTIMIZED", "OBJECT_STORE"})
void testKeyUpdatedShouldNotGetDeleted(BucketLayout bucketLayout)
throws IOException, InterruptedException, TimeoutException {
+ assumeTrue(stateSaveInternal != -1);
final String volumeName = getTestName();
final String bucketName = uniqueObjectName("bucket");
GenericTestUtils.LogCapturer log =
@@ -1049,12 +1971,13 @@ void testKeyUpdatedShouldNotGetDeleted(BucketLayout bucketLayout)
GenericTestUtils.LogCapturer requestLog =
GenericTestUtils.LogCapturer.captureLogs(
LoggerFactory.getLogger(OMKeysDeleteRequest.class));
- String prefix = "key";
+ String keyPrefix = "key";
+ String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : keyPrefix;
long initialDeletedKeyCount = getDeletedKeyCount();
long initialKeyCount = getKeyCount(bucketLayout);
// create keys
List<OmKeyArgs> keyList =
- createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, prefix, null);
+ createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, keyPrefix, null);
KeyLifecycleService.setInjectors(
Arrays.asList(new FaultInjectorImpl(), new FaultInjectorImpl()));
@@ -1062,7 +1985,7 @@ void testKeyUpdatedShouldNotGetDeleted(BucketLayout bucketLayout)
// create Lifecycle configuration
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
- createLifecyclePolicy(volumeName, bucketName, bucketLayout, prefix, null, date.toString(), true);
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true);
Thread.sleep(SERVICE_INTERVAL);
KeyLifecycleService.getInjector(0).resume();
@@ -1130,7 +2053,7 @@ void testPerformanceWithExpiredKeys(BucketLayout bucketLayout)
throws IOException, InterruptedException, TimeoutException {
final String volumeName = getTestName();
final String bucketName = uniqueObjectName("bucket");
- String prefix = "key";
+ String keyPrefix = "key";
long initialDeletedKeyCount = getDeletedKeyCount();
long initialKeyCount = getKeyCount(bucketLayout);
long initialKeyDeleted = metrics.getNumKeyDeleted().value();
@@ -1139,7 +2062,7 @@ void testPerformanceWithExpiredKeys(BucketLayout bucketLayout)
final int keyCount = 10;
// create keys
List<OmKeyArgs> keyList =
- createKeys(volumeName, bucketName, bucketLayout, keyCount, 1, prefix, null);
+ createKeys(volumeName, bucketName, bucketLayout, keyCount, 1, keyPrefix, null);
// check there are keys in keyTable
assertEquals(keyCount, keyList.size());
GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == keyCount,
@@ -1148,12 +2071,13 @@ void testPerformanceWithExpiredKeys(BucketLayout bucketLayout)
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
List<OmLCRule> ruleList = new ArrayList<>();
+ String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : keyPrefix;
ruleList.add(new OmLCRule.Builder().setId(String.valueOf(OBJECT_ID_COUNTER.getAndIncrement()))
- .setEnabled(false).setPrefix(prefix)
+ .setEnabled(false).setPrefix(rulePrefix)
.setAction(new OmLCExpiration.Builder().setDate(date.toString()).build())
.build());
ruleList.add(new OmLCRule.Builder().setId(String.valueOf(OBJECT_ID_COUNTER.getAndIncrement()))
- .setEnabled(true).setPrefix(prefix)
+ .setEnabled(true).setPrefix(rulePrefix)
.setAction(new OmLCExpiration.Builder().setDate(date.toString()).build())
.build());
createLifecyclePolicy(volumeName, bucketName, bucketLayout, ruleList);
@@ -1251,9 +2175,8 @@ void testPerformanceWithNestedDir(BucketLayout bucketLayout, String prefix)
GenericTestUtils.waitFor(() -> metrics.getNumKeyDeleted().value() - initialKeyDeleted == keyCount,
WAIT_CHECK_INTERVAL, 5000);
assertEquals(0, metrics.getNumDirIterated().value() - initialDirIterated);
- GenericTestUtils.waitFor(() ->
- metrics.getNumDirDeleted().value() - initialDirDeleted == (bucketLayout == FILE_SYSTEM_OPTIMIZED ? 1 : 0),
- WAIT_CHECK_INTERVAL, 10000);
+ GenericTestUtils.waitFor(() -> metrics.getNumDirDeleted().value() - initialDirDeleted == 0,
+ WAIT_CHECK_INTERVAL, 5000);
deleteLifecyclePolicy(volumeName, bucketName);
}
@@ -1273,7 +2196,7 @@ void testListMaxSize(BucketLayout bucketLayout, boolean enableTrash) throws IOEx
TimeoutException, InterruptedException {
final String volumeName = getTestName();
final String bucketName = uniqueObjectName("bucket");
- String prefix = "key";
+ String keyPrefix = "key";
long initialDeletedKeyCount = getDeletedKeyCount();
long initialKeyCount = getKeyCount(bucketLayout);
long initialRenamedKeyCount = metrics.getNumKeyRenamed().value();
@@ -1282,7 +2205,7 @@ void testListMaxSize(BucketLayout bucketLayout, boolean enableTrash) throws IOEx
keyLifecycleService.setListMaxSize(maxListSize);
// create keys
List<OmKeyArgs> keyList =
- createKeys(volumeName, bucketName, bucketLayout, keyCount, 1, prefix, null);
+ createKeys(volumeName, bucketName, bucketLayout, keyCount, 1, keyPrefix, null);
// check there are keys in keyTable
Thread.sleep(SERVICE_INTERVAL);
assertEquals(keyCount, keyList.size());
@@ -1305,7 +2228,8 @@ void testListMaxSize(BucketLayout bucketLayout, boolean enableTrash) throws IOEx
// create Lifecycle configuration
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
- createLifecyclePolicy(volumeName, bucketName, bucketLayout, prefix, null, date.toString(), true);
+ String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : keyPrefix;
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true);
if (enableTrash && bucketLayout != OBJECT_STORE) {
GenericTestUtils.waitFor(() ->
@@ -1316,9 +2240,11 @@ void testListMaxSize(BucketLayout bucketLayout, boolean enableTrash) throws IOEx
(getDeletedKeyCount() - initialDeletedKeyCount) == keyCount, WAIT_CHECK_INTERVAL, 5000);
assertEquals(0, getKeyCount(bucketLayout) - initialKeyCount);
}
- GenericTestUtils.waitFor(() ->
- log.getOutput().contains("LimitedSizeList has reached maximum size " + maxListSize),
- WAIT_CHECK_INTERVAL, 5000);
+ if (stateSaveInternal != -1) {
+ GenericTestUtils.waitFor(() ->
+ log.getOutput().contains("LimitedSizeList has reached maximum size " + maxListSize),
+ WAIT_CHECK_INTERVAL, 5000);
+ }
GenericTestUtils.setLogLevel(KeyLifecycleService.getLog(), Level.INFO);
deleteLifecyclePolicy(volumeName, bucketName);
}
@@ -1364,67 +2290,98 @@ void testMoveToTrash(BucketLayout bucketLayout, String prefix) throws IOExceptio
ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
createLifecyclePolicy(volumeName, bucketName, bucketLayout, "", null, date.toString(), true);
- GenericTestUtils.waitFor(() ->
- (metrics.getNumKeyRenamed().value() - initialRenamedKeyCount) == KEY_COUNT, WAIT_CHECK_INTERVAL, 50000);
- assertEquals(0, getDeletedKeyCount() - initialDeletedKeyCount);
- if (bucketLayout == FILE_SYSTEM_OPTIMIZED) {
- // Legacy bucket doesn't have dir concept
+ try {
GenericTestUtils.waitFor(() ->
- metrics.getNumDirRenamed().value() - initialRenamedDirCount == (prefix.contains(OM_KEY_PREFIX) ? 1 : 0),
- WAIT_CHECK_INTERVAL, 5000);
- }
- deleteLifecyclePolicy(volumeName, bucketName);
- // verify trash directory has the right native ACLs
- List<KeyInfoWithVolumeContext> dirList = new ArrayList<>();
- if (bucketLayout == FILE_SYSTEM_OPTIMIZED) {
- dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX));
- dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX + OM_KEY_PREFIX + bucketOwner));
- dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX + OM_KEY_PREFIX + bucketOwner +
- OM_KEY_PREFIX + CURRENT));
- } else {
- dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX + OM_KEY_PREFIX));
- dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX + OM_KEY_PREFIX + bucketOwner + OM_KEY_PREFIX));
- dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX + OM_KEY_PREFIX + bucketOwner +
- OM_KEY_PREFIX + CURRENT + OM_KEY_PREFIX));
- }
- for (KeyInfoWithVolumeContext dir : dirList) {
- List<OzoneAcl> aclList = dir.getKeyInfo().getAcls();
- for (OzoneAcl acl : aclList) {
- if (acl.getType() == IAccessAuthorizer.ACLIdentityType.USER ||
- acl.getType() == IAccessAuthorizer.ACLIdentityType.GROUP) {
- assertEquals(bucketOwner, acl.getName());
- assertTrue(acl.getAclList().contains(ALL));
+ (metrics.getNumKeyRenamed().value() - initialRenamedKeyCount) == KEY_COUNT, WAIT_CHECK_INTERVAL, 5000);
+ assertEquals(0, getDeletedKeyCount() - initialDeletedKeyCount);
+ if (bucketLayout == FILE_SYSTEM_OPTIMIZED) {
+ // Legacy bucket doesn't have dir concept
+ GenericTestUtils.waitFor(() ->
+ metrics.getNumDirRenamed().value() - initialRenamedDirCount == (prefix.contains(OM_KEY_PREFIX) ?
+ 1 : 0), WAIT_CHECK_INTERVAL, 5000);
+ }
+
+ // verify that trash directory has the right native ACLs
+ List<KeyInfoWithVolumeContext> dirList = new ArrayList<>();
+ if (bucketLayout == FILE_SYSTEM_OPTIMIZED) {
+ dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX));
+ dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX + OM_KEY_PREFIX + bucketOwner));
+ dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX + OM_KEY_PREFIX + bucketOwner +
+ OM_KEY_PREFIX + CURRENT));
+ } else {
+ dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX + OM_KEY_PREFIX));
+ dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX + OM_KEY_PREFIX + bucketOwner + OM_KEY_PREFIX));
+ dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX + OM_KEY_PREFIX + bucketOwner +
+ OM_KEY_PREFIX + CURRENT + OM_KEY_PREFIX));
+ }
+ for (KeyInfoWithVolumeContext dir : dirList) {
+ List<OzoneAcl> aclList = dir.getKeyInfo().getAcls();
+ for (OzoneAcl acl : aclList) {
+ if (acl.getType() == IAccessAuthorizer.ACLIdentityType.USER ||
+ acl.getType() == IAccessAuthorizer.ACLIdentityType.GROUP) {
+ assertEquals(bucketOwner, acl.getName());
+ assertTrue(acl.getAclList().contains(ALL));
+ }
}
}
+
+ // keys under trash directory is counted in getKeyCount()
+ if (bucketLayout == FILE_SYSTEM_OPTIMIZED) {
+ assertEquals(KEY_COUNT, getKeyCount(bucketLayout) - initialKeyCount);
+ } else {
+ // For legacy bucket, trash directories along .Trash/user-test/Current are in key table too.
+ assertEquals(KEY_COUNT + (prefix.contains(OM_KEY_PREFIX) ? 4 : 3),
+ getKeyCount(bucketLayout) - initialKeyCount);
+ }
+ } finally {
+ deleteLifecyclePolicy(volumeName, bucketName);
}
+ }
- GenericTestUtils.LogCapturer log =
- GenericTestUtils.LogCapturer.captureLogs(
- LoggerFactory.getLogger(KeyLifecycleService.class));
+ @ParameterizedTest
+ @MethodSource("parameters7")
+ void testMoveToTrashWithTrashPrefix(BucketLayout bucketLayout, String prefix) throws IOException,
+ TimeoutException, InterruptedException {
+ final String volumeName = getTestName();
+ final String bucketName = uniqueObjectName("bucket");
+ long initialKeyCount = getKeyCount(bucketLayout);
+ // create keys
+ String bucketOwner = UserGroupInformation.getCurrentUser().getShortUserName() + "-test";
+ List<OmKeyArgs> keyList =
+ createKeys(volumeName, bucketName, bucketLayout, bucketOwner, KEY_COUNT, 1, prefix, null);
+ // check there are keys in keyTable
+ Thread.sleep(SERVICE_INTERVAL);
+ assertEquals(KEY_COUNT, keyList.size());
+ GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == KEY_COUNT,
+ WAIT_CHECK_INTERVAL, 1000);
- // keys under trash directory is counted in getKeyCount()
- if (bucketLayout == FILE_SYSTEM_OPTIMIZED) {
- assertEquals(KEY_COUNT, getKeyCount(bucketLayout) - initialKeyCount);
- } else {
- // For legacy bucket, trash directories along .Trash/user-test/Current are in key table too.
- assertEquals(KEY_COUNT + (prefix.contains(OM_KEY_PREFIX) ? 4 : 3), getKeyCount(bucketLayout) - initialKeyCount);
- }
- // create new policy to test rule with prefix ".Trash/" is ignored during lifecycle evaluation
- now = ZonedDateTime.now(ZoneOffset.UTC);
+ // enabled trash
+ final float trashInterval = 0.5f; // 30 seconds, 0.5 * (60 * 1000) ms
+ conf.setFloat(FS_TRASH_INTERVAL_KEY, trashInterval);
+ FileSystem fs = SecurityUtil.doAsLoginUser(
+ (PrivilegedExceptionAction<FileSystem>)
+ () -> new TrashOzoneFileSystem(om));
+ keyLifecycleService.setOzoneTrash(new OzoneTrash(fs, conf, om));
+
+ // create a new policy to test rule with prefix ".Trash/" is ignored during lifecycle evaluation
+ ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
final String expiredDate = now.plusSeconds(EXPIRE_SECONDS).toString();
assertThrowsExactly(OMException.class, () -> createLifecyclePolicy(
volumeName, bucketName, bucketLayout, TRASH_PREFIX + OM_KEY_PREFIX, null, expiredDate, true));
- // create new policy to test rule with prefix ".Trash" is ignored during lifecycle evaluation
+ // create a new policy to test rule with prefix ".Trash" is ignored during lifecycle evaluation
assertThrowsExactly(OMException.class, () -> createLifecyclePolicy(
volumeName, bucketName, bucketLayout, TRASH_PREFIX,
null, expiredDate, true));
- // create new policy to test rule with prefix ".Tras" is ignored during lifecycle evaluation
+ // create a new policy to test rule with prefix ".Tras/" is ignored during lifecycle evaluation
now = ZonedDateTime.now(ZoneOffset.UTC);
- date = now.plusSeconds(EXPIRE_SECONDS);
- createLifecyclePolicy(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED, ".Tras", null, date.toString(), true);
+ ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
+ createLifecyclePolicy(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED, ".Tras/", null, date.toString(), true);
+ GenericTestUtils.LogCapturer log =
+ GenericTestUtils.LogCapturer.captureLogs(
+ LoggerFactory.getLogger(KeyLifecycleService.class));
GenericTestUtils.waitFor(
() -> log.getOutput().contains("No expired keys/dirs found/remained for bucket"), WAIT_CHECK_INTERVAL, 5000);
deleteLifecyclePolicy(volumeName, bucketName);
@@ -1442,23 +2399,27 @@ void testMoveToTrash(BucketLayout bucketLayout, String prefix) throws IOExceptio
public Stream<Arguments> parameters8() {
return Stream.of(
- arguments("dir1/dir2/dir3/key", null, "dir1/dir", "dir1/dir2/dir3", KEY_COUNT, 2, false),
- arguments("dir1/dir2/dir3/key", null, "dir1/dir", "dir1/dir2/dir3", KEY_COUNT, 0, true),
- arguments("dir1/dir2/dir3/key", "dir1/dir4/dir5/key", "dir1/dir", "dir1/dir2/dir3", KEY_COUNT * 2, 4, false),
- arguments("dir1/dir2/dir3/key", "dir1/dir4/dir5/key", "dir1/dir", "dir1/dir2/dir3", KEY_COUNT * 2, 2, true),
- arguments("dir1/dir2/dir3/key", "dir1/dir22/dir5/key", "dir1/dir2/", "dir1/dir2/dir3", KEY_COUNT, 2, false),
- arguments("dir1/dir2/dir3/key", "dir1/dir22/dir5/key", "dir1/dir2/", "dir1/dir2/dir3", KEY_COUNT, 0, true),
- arguments("dir1/dir2/dir3/key", "dir1/dir22/dir5/key", "dir1/dir2", "dir1/dir2/dir3",
- KEY_COUNT * 2, 4, false),
- arguments("dir1/dir2/dir3/key", "dir1/dir22/dir5/key", "dir1/dir2", "dir1/dir2/dir3", KEY_COUNT * 2, 2, true),
- arguments("dir1/dir2/dir3/key", "dir1/dir4/dir5/key", "dir", "dir1/dir2/dir3", KEY_COUNT * 2, 5, false),
- arguments("dir1/dir2/dir3/key", "dir1/dir4/dir5/key", "dir", "dir1/dir2/dir3", KEY_COUNT * 2, 2, true),
- arguments("dir1/dir2/dir3/key", "dir11/dir4/dir5/key", "dir1/", "dir1/dir2/dir3", KEY_COUNT, 3, false),
- arguments("dir1/dir2/dir3/key", "dir11/dir4/dir5/key", "dir1/", "dir1/dir2/dir3", KEY_COUNT, 0, true),
- arguments("dir1/dir2/dir3/key", "dir11/dir4/dir5/key", "dir1", "dir1/dir2/dir3", KEY_COUNT * 2, 6, false),
- arguments("dir1/dir2/dir3/key", "dir11/dir4/dir5/key", "dir1", "dir1/dir2/dir3", KEY_COUNT * 2, 3, true),
+ // dir3 and keys under dir3 deleted
+ arguments("dir1/dir2/dir3/key", null, "dir1/dir2/", "dir1/dir2/dir3/", KEY_COUNT, 1, false),
+ // no dir, but keys under dir3 deleted
+ arguments("dir1/dir2/dir3/key", null, "dir1/dir2/", "dir1/dir2/dir3/", KEY_COUNT, 0, true),
+ // dir3 dir5, and all keys under dir3 and dir5 deleted
+ arguments("dir1/dir2/dir3/key", "dir1/dir2/dir5/key", "dir1/dir2/", "dir1/dir2/dir3",
+ KEY_COUNT * 2, 2, false),
+ // dir5, and all keys under dir3 and dir5 deleted
+ arguments("dir1/dir2/dir3/key", "dir1/dir2/dir5/key", "dir1/dir2/", "dir1/dir2/dir3", KEY_COUNT * 2, 1, true),
+ // dir2 dir3 dir4 dir5, and all keys under dir3 and dir5 deleted
+ arguments("dir1/dir2/dir3/key", "dir1/dir4/dir5/key", "dir1/", "dir1/dir2/dir3", KEY_COUNT * 2, 4, false),
+ // dir4 dir5, and all keys under dir3 and dir5 deleted
+ arguments("dir1/dir2/dir3/key", "dir1/dir4/dir5/key", "dir1/", "dir1/dir2/dir3", KEY_COUNT * 2, 2, true),
+ // dir1 - dir5, and all keys deleted
arguments("dir1/dir2/dir3/key", "dir1/dir4/dir5/key", "", "dir1/dir2/dir3", KEY_COUNT * 2, 5, false),
- arguments("dir1/dir2/dir3/key", "dir1/dir4/dir5/key", "", "dir1/dir2/dir3", KEY_COUNT * 2, 2, true));
+ // dir4 dir5, and all keys deleted
+ arguments("dir1/dir2/dir3/key", "dir1/dir4/dir5/key", "", "dir1/dir2/dir3", KEY_COUNT * 2, 2, true),
+ // dir4 dir5, and all keys under dir5 deleted
+ arguments("dir11/dir4/dir5/key", "dir1/dir2/dir3/key", "dir11/", "dir11/dir4/dir5", KEY_COUNT, 2, false),
+ // no dir, but all keys under dir11 deleted
+ arguments("dir11/dir4/dir5/key", "dir1/dir2/dir3/key", "dir11/", "dir11/dir4/dir5", KEY_COUNT, 0, true));
}
@ParameterizedTest
@@ -1506,7 +2467,7 @@ void testMultipleDirectoriesMatched(String keyPrefix1, String keyPrefix2, String
LOG.info("expiry date {}", date.toInstant());
ZonedDateTime endDate = date.plus(SERVICE_INTERVAL, ChronoUnit.MILLIS);
- GenericTestUtils.waitFor(() -> endDate.isBefore(ZonedDateTime.now(ZoneOffset.UTC)), WAIT_CHECK_INTERVAL, 10000);
+ GenericTestUtils.waitFor(() -> endDate.isBefore(ZonedDateTime.now(ZoneOffset.UTC)), WAIT_CHECK_INTERVAL, 5000);
// rename a key under directory to change directory's Modification time
if (updateDirModificationTime) {
@@ -1555,7 +2516,7 @@ void testGetLifecycleServiceStatus() throws Exception {
ZonedDateTime date = ZonedDateTime.now(ZoneOffset.UTC).plusSeconds(EXPIRE_SECONDS);
KeyLifecycleService.setInjectors(
Arrays.asList(new FaultInjectorImpl(), new FaultInjectorImpl()));
- createLifecyclePolicy(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED, prefix, null, date.toString(), true);
+ createLifecyclePolicy(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED, "", null, date.toString(), true);
Thread.sleep(SERVICE_INTERVAL + 100);
// Verify service is running and processing the bucket
@@ -1959,6 +2920,7 @@ public Stream<Arguments> parameters1() {
@MethodSource("parameters1")
void testKeyDeletedOrRenamed(BucketLayout bucketLayout, boolean deleted)
throws IOException, InterruptedException, TimeoutException {
+ assumeTrue(stateSaveInternal != -1);
final String volumeName = getTestName();
final String bucketName = uniqueObjectName("bucket");
GenericTestUtils.LogCapturer log =
@@ -1967,12 +2929,13 @@ void testKeyDeletedOrRenamed(BucketLayout bucketLayout, boolean deleted)
GenericTestUtils.LogCapturer requestLog =
GenericTestUtils.LogCapturer.captureLogs(
LoggerFactory.getLogger(OMKeysDeleteRequest.class));
- String prefix = "key";
+ String keyPrefix = "key";
+ String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : keyPrefix;
long initialDeletedKeyCount = getDeletedKeyCount();
long initialKeyCount = getKeyCount(bucketLayout);
// create keys
List<OmKeyArgs> keyList =
- createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, prefix, null);
+ createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, keyPrefix, null);
KeyLifecycleService.setInjectors(
Arrays.asList(new FaultInjectorImpl(), new FaultInjectorImpl()));
@@ -1980,13 +2943,14 @@ void testKeyDeletedOrRenamed(BucketLayout bucketLayout, boolean deleted)
// create Lifecycle configuration
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS);
- createLifecyclePolicy(volumeName, bucketName, bucketLayout, prefix, null, date.toString(), true);
+ createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true);
Thread.sleep(SERVICE_INTERVAL);
KeyLifecycleService.getInjector(0).resume();
GenericTestUtils.waitFor(
() -> log.getOutput().contains(KEY_COUNT + " expired keys and 0 expired dirs found"),
WAIT_CHECK_INTERVAL, 10000);
+
OmKeyArgs key = keyList.get(ThreadLocalRandom.current().nextInt(1, keyList.size()));
// delete/rename another key before send deletion requests
if (deleted) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org