You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@james.apache.org by rc...@apache.org on 2026/10/19 14:54:44 UTC
(james-project) branch feature/s3-object-compaction updated (f21ee59230 -> b9ea8e45d9)
This is an automated email from the ASF dual-hosted git repository.
Arsnael pushed a change to branch feature/s3-object-compaction
in repository https://gitbox.apache.org/repos/asf/james-project.git
from f21ee59230 [BUILD] Distributed APP: run tests in 2 surefire forks
new 2f01f39f0a JAMES-4231 Exclude chunk objects from BloomFilter GC
new 04d11a2392 JAMES-4231 Make string chunk marker helper private
new f3772027e0 JAMES-4231 Recognize MinIO chunk IDs and unrestricted identifiers
new 5d179aba53 JAMES-4231 Explain chunk pattern with named regex components
new b9ea8e45d9 JAMES-4231 Delimit chunk marker from object identifier
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/james/blob/api/ChunkMarker.java | 51 +++++++++++++++
.../org/apache/james/blob/api/ChunkMarkerTest.java | 75 ++++++++++++++++++++++
.../blob/deduplication/BloomFilterGCAlgorithm.java | 2 +
.../BloomFilterGCAlgorithmContract.java | 63 ++++++++++++++++++
4 files changed, 191 insertions(+)
create mode 100644 server/blob/blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java
create mode 100644 server/blob/blob-api/src/test/java/org/apache/james/blob/api/ChunkMarkerTest.java
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org
(james-project) 02/05: JAMES-4231 Make string chunk marker helper private
Posted by rc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Arsnael pushed a commit to branch feature/s3-object-compaction
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 04d11a23920553b71c902f2ef13a82d8872cdca0
Author: Hesanda Liyanage <13...@users.noreply.github.com>
AuthorDate: Fri Oct 16 09:35:10 2026 +0530
JAMES-4231 Make string chunk marker helper private
---
.../blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java | 2 +-
.../src/test/java/org/apache/james/blob/api/ChunkMarkerTest.java | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java
index 63cbba2a60..c31f9f86ae 100644
--- a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java
+++ b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java
@@ -38,7 +38,7 @@ public final class ChunkMarker {
return blobId != null && isChunkId(blobId.asString());
}
- public static boolean isChunkId(String blobId) {
+ private static boolean isChunkId(String blobId) {
return blobId != null && CHUNK_ID_PATTERN.matcher(blobId).matches();
}
}
diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/ChunkMarkerTest.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/ChunkMarkerTest.java
index 270c358588..b0ce01a42f 100644
--- a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/ChunkMarkerTest.java
+++ b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/ChunkMarkerTest.java
@@ -30,13 +30,13 @@ class ChunkMarkerTest {
@ParameterizedTest
@MethodSource("chunkIds")
void shouldRecognizeChunkIds(String blobId) {
- assertThat(ChunkMarker.isChunkId(blobId)).isTrue();
+ assertThat(ChunkMarker.isChunkId(new PlainBlobId(blobId))).isTrue();
}
@ParameterizedTest
@MethodSource("nonChunkIds")
void shouldRejectNonChunkIds(String blobId) {
- assertThat(ChunkMarker.isChunkId(blobId)).isFalse();
+ assertThat(ChunkMarker.isChunkId(new PlainBlobId(blobId))).isFalse();
}
private static Stream<String> chunkIds() {
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org
(james-project) 03/05: JAMES-4231 Recognize MinIO chunk IDs and unrestricted identifiers
Posted by rc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Arsnael pushed a commit to branch feature/s3-object-compaction
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit f3772027e0ca647298393a826b2891fde2122bce
Author: Hesanda Liyanage <13...@users.noreply.github.com>
AuthorDate: Sat Oct 17 14:46:04 2026 +0530
JAMES-4231 Recognize MinIO chunk IDs and unrestricted identifiers
---
.../org/apache/james/blob/api/ChunkMarker.java | 9 ++++----
.../org/apache/james/blob/api/ChunkMarkerTest.java | 19 +++++++++++++----
.../blob/deduplication/BloomFilterGCAlgorithm.java | 2 +-
.../BloomFilterGCAlgorithmContract.java | 24 ++++++++++++++++++++++
4 files changed, 45 insertions(+), 9 deletions(-)
diff --git a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java
index c31f9f86ae..e17b2d1618 100644
--- a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java
+++ b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java
@@ -23,13 +23,14 @@ import java.util.regex.Pattern;
/**
* Identifies physical chunk object identifiers and their virtual slot references.
*
- * <p>The compacted-object format is {@code <family>_<generation>_chunk<random-part>} for the physical object and
- * {@code <physical-object>~<offset>~<length>} for a virtual slot. This shared marker deliberately contains no
- * chunk-format parsing so storage strategies can recognize chunk IDs without depending on the compaction module.</p>
+ * <p>Physical objects use {@code <family>_<generation>_chunk<identifier>} or the MinIO layout
+ * {@code <family>/<generation>/chunk<identifier>}. The identifier is unrestricted and can also carry a virtual
+ * slot suffix. This only recognizes the chunk marker; parsing and validating a slot reference is the
+ * responsibility of the compaction module.</p>
*/
public final class ChunkMarker {
private static final Pattern CHUNK_ID_PATTERN = Pattern.compile(
- "^[1-9]\\d*_\\d+_chunk[A-Za-z0-9_-]{16,}(?:~\\d+~\\d+)?$");
+ "^[1-9]\\d*(?:_\\d+_chunk|/\\d+/chunk).*$");
private ChunkMarker() {
}
diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/ChunkMarkerTest.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/ChunkMarkerTest.java
index b0ce01a42f..b562d61edc 100644
--- a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/ChunkMarkerTest.java
+++ b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/ChunkMarkerTest.java
@@ -43,16 +43,27 @@ class ChunkMarkerTest {
return Stream.of(
"1_0_chunk0123456789abcdef",
"12_34_chunk0123456789abcdef_ABC-xyz",
- "1_0_chunk0123456789abcdef~0~123");
+ "1_0_chunk0123456789abcdef~0~123",
+ "1_690_chunkshort",
+ "1/690/chunkshort",
+ "12/12345/chunk550e8400-e29b-41d4-a716-446655440000",
+ "1_12345_chunk550e8400-e29b-41d4-a716-446655440000",
+ "1/690/chunkidentifier.with+other=characters~0~123",
+ "1_690_chunkidentifier.with+other=characters",
+ "1/690/chunk",
+ "1_690_chunk");
}
private static Stream<String> nonChunkIds() {
return Stream.of(
"0_0_chunk0123456789abcdef",
"1_-1_chunk0123456789abcdef",
- "1_0_chunk0123456789abcde",
- "1_0_chunk0123456789abcdef~1",
- "1_0_chunk0123456789abcdef~1~-1",
+ "0/690/chunkidentifier",
+ "1/-1/chunkidentifier",
+ "1/690/ordinary-blob",
+ "1/690/hash_chunk_identifier",
+ "1/690_chunkidentifier",
+ "1_690/chunkidentifier",
"1_0_not-a-chunk0123456789abcdef",
"a_chunk0123456789abcdef",
"plain-blob-id");
diff --git a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithm.java b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithm.java
index ac54bab59d..3fc00a30a1 100644
--- a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithm.java
+++ b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithm.java
@@ -275,6 +275,7 @@ public class BloomFilterGCAlgorithm {
private Mono<Result> gc(BloomFilter<CharSequence> bloomFilter, BucketName bucketName, Context context, int deletionWindowSize) {
return Flux.from(blobStoreDAO.listBlobs(bucketName))
.doOnNext(blobId -> context.incrementBlobCount())
+ .filter(blobId -> !ChunkMarker.isChunkId(blobId))
.flatMap(blobId -> Mono.fromCallable(() -> blobIdFactory.parse(blobId.asString())))
.filter(blobId -> {
if (blobId instanceof GenerationAware generationAware) {
@@ -282,7 +283,6 @@ public class BloomFilterGCAlgorithm {
}
return false;
})
- .filter(blobId -> !ChunkMarker.isChunkId(blobId))
.filter(blobId -> !bloomFilter.mightContain(salt + blobId.asString()))
.window(deletionWindowSize)
.flatMap(blobIdFlux -> handlePagedDeletion(bucketName, context, blobIdFlux), DEFAULT_CONCURRENCY)
diff --git a/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithmContract.java b/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithmContract.java
index f919990f8d..307f3d9169 100644
--- a/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithmContract.java
+++ b/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithmContract.java
@@ -183,6 +183,30 @@ public interface BloomFilterGCAlgorithmContract {
.isInstanceOf(ObjectNotFoundException.class);
}
+ @Test
+ default void gcShouldPreserveMinIOChunksAndRemoveOrdinaryMinIOBlobs() {
+ BlobStoreDAO store = blobStoreDAO();
+ when(BLOB_REFERENCE_SOURCE.listReferencedBlobs()).thenReturn(Flux.empty());
+ MinIOGenerationAwareBlobId.Factory minIOFactory = new MinIOGenerationAwareBlobId.Factory(
+ CLOCK, GENERATION_AWARE_BLOB_ID_CONFIGURATION, BLOB_ID_FACTORY);
+ BlobId chunk = minIOFactory.parse("1/12/chunk" + UUID.randomUUID());
+ BlobId ordinary = minIOFactory.parse("1/12/ordinary-blob");
+ Mono.from(store.save(DEFAULT_BUCKET, chunk, BlobStoreDAO.BytesBlob.of("chunk"))).block();
+ Mono.from(store.save(DEFAULT_BUCKET, ordinary, BlobStoreDAO.BytesBlob.of("ordinary"))).block();
+ CLOCK.setInstant(NOW.plusMonths(2).toInstant());
+
+ BloomFilterGCAlgorithm algorithm = new BloomFilterGCAlgorithm(BLOB_REFERENCE_SOURCE, store, minIOFactory,
+ GENERATION_AWARE_BLOB_ID_CONFIGURATION, CLOCK);
+ Context context = new Context(EXPECTED_BLOB_COUNT, ASSOCIATED_PROBABILITY);
+
+ assertThat(algorithm.gc(EXPECTED_BLOB_COUNT, DELETION_WINDOW_SIZE, ASSOCIATED_PROBABILITY, DEFAULT_BUCKET, context).block())
+ .isEqualTo(Task.Result.COMPLETED);
+ assertThat(Mono.from(store.readBytes(DEFAULT_BUCKET, chunk)).block().payload())
+ .isEqualTo("chunk".getBytes());
+ assertThatThrownBy(() -> store.read(DEFAULT_BUCKET, ordinary))
+ .isInstanceOf(ObjectNotFoundException.class);
+ }
+
@RepeatedTest(10)
default void gcShouldNotRemoveReferencedBlob() {
BlobStore blobStore = blobStore();
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org
(james-project) 04/05: JAMES-4231 Explain chunk pattern with named regex components
Posted by rc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Arsnael pushed a commit to branch feature/s3-object-compaction
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 5d179aba53f90662cd0a1c529d5350e714ff4554
Author: Hesanda Liyanage <13...@users.noreply.github.com>
AuthorDate: Sat Oct 17 14:47:23 2026 +0530
JAMES-4231 Explain chunk pattern with named regex components
---
.../src/main/java/org/apache/james/blob/api/ChunkMarker.java | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java
index e17b2d1618..54aae33ad2 100644
--- a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java
+++ b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java
@@ -29,8 +29,14 @@ import java.util.regex.Pattern;
* responsibility of the compaction module.</p>
*/
public final class ChunkMarker {
+ private static final String FAMILY = "[1-9]\\d*";
+ private static final String GENERATION = "\\d+";
+ private static final String CHUNK_MARKER = "chunk";
+ private static final String REMAINDER = ".*";
+ private static final String UNDERSCORE_LAYOUT = "_" + GENERATION + "_" + CHUNK_MARKER;
+ private static final String MINIO_LAYOUT = "/" + GENERATION + "/" + CHUNK_MARKER;
private static final Pattern CHUNK_ID_PATTERN = Pattern.compile(
- "^[1-9]\\d*(?:_\\d+_chunk|/\\d+/chunk).*$");
+ "^" + FAMILY + "(?:" + UNDERSCORE_LAYOUT + "|" + MINIO_LAYOUT + ")" + REMAINDER + "$");
private ChunkMarker() {
}
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org
(james-project) 01/05: JAMES-4231 Exclude chunk objects from BloomFilter GC
Posted by rc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Arsnael pushed a commit to branch feature/s3-object-compaction
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 2f01f39f0ad947a25c4e1dec894e4fc56da5e6ad
Author: Hesanda Liyanage <13...@users.noreply.github.com>
AuthorDate: Sat Oct 10 17:05:36 2026 +0530
JAMES-4231 Exclude chunk objects from BloomFilter GC
---
.../org/apache/james/blob/api/ChunkMarker.java | 44 ++++++++++++++++
.../org/apache/james/blob/api/ChunkMarkerTest.java | 60 ++++++++++++++++++++++
.../blob/deduplication/BloomFilterGCAlgorithm.java | 2 +
.../BloomFilterGCAlgorithmContract.java | 39 ++++++++++++++
4 files changed, 145 insertions(+)
diff --git a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java
new file mode 100644
index 0000000000..63cbba2a60
--- /dev/null
+++ b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java
@@ -0,0 +1,44 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.blob.api;
+
+import java.util.regex.Pattern;
+
+/**
+ * Identifies physical chunk object identifiers and their virtual slot references.
+ *
+ * <p>The compacted-object format is {@code <family>_<generation>_chunk<random-part>} for the physical object and
+ * {@code <physical-object>~<offset>~<length>} for a virtual slot. This shared marker deliberately contains no
+ * chunk-format parsing so storage strategies can recognize chunk IDs without depending on the compaction module.</p>
+ */
+public final class ChunkMarker {
+ private static final Pattern CHUNK_ID_PATTERN = Pattern.compile(
+ "^[1-9]\\d*_\\d+_chunk[A-Za-z0-9_-]{16,}(?:~\\d+~\\d+)?$");
+
+ private ChunkMarker() {
+ }
+
+ public static boolean isChunkId(BlobId blobId) {
+ return blobId != null && isChunkId(blobId.asString());
+ }
+
+ public static boolean isChunkId(String blobId) {
+ return blobId != null && CHUNK_ID_PATTERN.matcher(blobId).matches();
+ }
+}
diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/ChunkMarkerTest.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/ChunkMarkerTest.java
new file mode 100644
index 0000000000..270c358588
--- /dev/null
+++ b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/ChunkMarkerTest.java
@@ -0,0 +1,60 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.blob.api;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.stream.Stream;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+
+class ChunkMarkerTest {
+ @ParameterizedTest
+ @MethodSource("chunkIds")
+ void shouldRecognizeChunkIds(String blobId) {
+ assertThat(ChunkMarker.isChunkId(blobId)).isTrue();
+ }
+
+ @ParameterizedTest
+ @MethodSource("nonChunkIds")
+ void shouldRejectNonChunkIds(String blobId) {
+ assertThat(ChunkMarker.isChunkId(blobId)).isFalse();
+ }
+
+ private static Stream<String> chunkIds() {
+ return Stream.of(
+ "1_0_chunk0123456789abcdef",
+ "12_34_chunk0123456789abcdef_ABC-xyz",
+ "1_0_chunk0123456789abcdef~0~123");
+ }
+
+ private static Stream<String> nonChunkIds() {
+ return Stream.of(
+ "0_0_chunk0123456789abcdef",
+ "1_-1_chunk0123456789abcdef",
+ "1_0_chunk0123456789abcde",
+ "1_0_chunk0123456789abcdef~1",
+ "1_0_chunk0123456789abcdef~1~-1",
+ "1_0_not-a-chunk0123456789abcdef",
+ "a_chunk0123456789abcdef",
+ "plain-blob-id");
+ }
+}
diff --git a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithm.java b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithm.java
index b7523c2af8..ac54bab59d 100644
--- a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithm.java
+++ b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithm.java
@@ -34,6 +34,7 @@ import org.apache.james.blob.api.BlobId;
import org.apache.james.blob.api.BlobReferenceSource;
import org.apache.james.blob.api.BlobStoreDAO;
import org.apache.james.blob.api.BucketName;
+import org.apache.james.blob.api.ChunkMarker;
import org.apache.james.task.Task;
import org.apache.james.task.Task.Result;
import org.slf4j.Logger;
@@ -281,6 +282,7 @@ public class BloomFilterGCAlgorithm {
}
return false;
})
+ .filter(blobId -> !ChunkMarker.isChunkId(blobId))
.filter(blobId -> !bloomFilter.mightContain(salt + blobId.asString()))
.window(deletionWindowSize)
.flatMap(blobIdFlux -> handlePagedDeletion(bucketName, context, blobIdFlux), DEFAULT_CONCURRENCY)
diff --git a/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithmContract.java b/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithmContract.java
index 39397f1f14..f919990f8d 100644
--- a/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithmContract.java
+++ b/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithmContract.java
@@ -144,6 +144,45 @@ public interface BloomFilterGCAlgorithmContract {
.isNotNull();
}
+ @Test
+ default void gcShouldNotRemoveChunkObjectsEvenWhenUnreferenced() {
+ BlobStoreDAO blobStoreDAO = blobStoreDAO();
+ when(BLOB_REFERENCE_SOURCE.listReferencedBlobs()).thenReturn(Flux.empty());
+
+ BlobId normalBlob = GENERATION_AWARE_BLOB_ID_FACTORY.of(UUID.randomUUID().toString());
+ BlobId chunkBlob = BLOB_ID_FACTORY.parse("1_1_chunk0123456789abcdef");
+
+ Mono.from(blobStoreDAO.save(DEFAULT_BUCKET, normalBlob, BlobStoreDAO.BytesBlob.of("normal".getBytes()))).block();
+ Mono.from(blobStoreDAO.save(DEFAULT_BUCKET, chunkBlob, BlobStoreDAO.BytesBlob.of("chunk".getBytes()))).block();
+
+ CLOCK.setInstant(NOW.plusMonths(2).toInstant());
+
+ Context context = new Context(EXPECTED_BLOB_COUNT, ASSOCIATED_PROBABILITY);
+ Mono.from(bloomFilterGCAlgorithm().gc(EXPECTED_BLOB_COUNT, DELETION_WINDOW_SIZE, ASSOCIATED_PROBABILITY, DEFAULT_BUCKET, context)).block();
+
+ assertThatThrownBy(() -> blobStoreDAO.read(DEFAULT_BUCKET, normalBlob))
+ .isInstanceOf(ObjectNotFoundException.class);
+ assertThat(Mono.from(blobStoreDAO.readBytes(DEFAULT_BUCKET, chunkBlob)).block().payload())
+ .isEqualTo("chunk".getBytes());
+ }
+
+ @Test
+ default void gcShouldRemoveBlobIdsThatOnlyContainTheChunkSubstring() {
+ BlobStoreDAO blobStoreDAO = blobStoreDAO();
+ when(BLOB_REFERENCE_SOURCE.listReferencedBlobs()).thenReturn(Flux.empty());
+
+ BlobId nonChunkBlob = GENERATION_AWARE_BLOB_ID_FACTORY.of("hash_chunk_but_not_a_virtual_address");
+ Mono.from(blobStoreDAO.save(DEFAULT_BUCKET, nonChunkBlob, BlobStoreDAO.BytesBlob.of("ordinary".getBytes()))).block();
+
+ CLOCK.setInstant(NOW.plusMonths(2).toInstant());
+
+ Context context = new Context(EXPECTED_BLOB_COUNT, ASSOCIATED_PROBABILITY);
+ Mono.from(bloomFilterGCAlgorithm().gc(EXPECTED_BLOB_COUNT, DELETION_WINDOW_SIZE, ASSOCIATED_PROBABILITY, DEFAULT_BUCKET, context)).block();
+
+ assertThatThrownBy(() -> blobStoreDAO.read(DEFAULT_BUCKET, nonChunkBlob))
+ .isInstanceOf(ObjectNotFoundException.class);
+ }
+
@RepeatedTest(10)
default void gcShouldNotRemoveReferencedBlob() {
BlobStore blobStore = blobStore();
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org
(james-project) 05/05: JAMES-4231 Delimit chunk marker from object identifier
Posted by rc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Arsnael pushed a commit to branch feature/s3-object-compaction
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit b9ea8e45d93233920114edcbcc22a8879c4e2e6b
Author: Hesanda Liyanage <13...@users.noreply.github.com>
AuthorDate: Mon Oct 19 12:35:25 2026 +0530
JAMES-4231 Delimit chunk marker from object identifier
---
.../org/apache/james/blob/api/ChunkMarker.java | 8 ++---
.../org/apache/james/blob/api/ChunkMarkerTest.java | 38 ++++++++++++----------
.../BloomFilterGCAlgorithmContract.java | 4 +--
3 files changed, 27 insertions(+), 23 deletions(-)
diff --git a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java
index 54aae33ad2..0a6c82dd9f 100644
--- a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java
+++ b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/ChunkMarker.java
@@ -23,8 +23,8 @@ import java.util.regex.Pattern;
/**
* Identifies physical chunk object identifiers and their virtual slot references.
*
- * <p>Physical objects use {@code <family>_<generation>_chunk<identifier>} or the MinIO layout
- * {@code <family>/<generation>/chunk<identifier>}. The identifier is unrestricted and can also carry a virtual
+ * <p>Physical objects use {@code <family>_<generation>_chunk_<identifier>} or the MinIO layout
+ * {@code <family>/<generation>/chunk/<identifier>}. The identifier is unrestricted and can also carry a virtual
* slot suffix. This only recognizes the chunk marker; parsing and validating a slot reference is the
* responsibility of the compaction module.</p>
*/
@@ -33,8 +33,8 @@ public final class ChunkMarker {
private static final String GENERATION = "\\d+";
private static final String CHUNK_MARKER = "chunk";
private static final String REMAINDER = ".*";
- private static final String UNDERSCORE_LAYOUT = "_" + GENERATION + "_" + CHUNK_MARKER;
- private static final String MINIO_LAYOUT = "/" + GENERATION + "/" + CHUNK_MARKER;
+ private static final String UNDERSCORE_LAYOUT = "_" + GENERATION + "_" + CHUNK_MARKER + "_";
+ private static final String MINIO_LAYOUT = "/" + GENERATION + "/" + CHUNK_MARKER + "/";
private static final Pattern CHUNK_ID_PATTERN = Pattern.compile(
"^" + FAMILY + "(?:" + UNDERSCORE_LAYOUT + "|" + MINIO_LAYOUT + ")" + REMAINDER + "$");
diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/ChunkMarkerTest.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/ChunkMarkerTest.java
index b562d61edc..dd36172a1b 100644
--- a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/ChunkMarkerTest.java
+++ b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/ChunkMarkerTest.java
@@ -41,29 +41,33 @@ class ChunkMarkerTest {
private static Stream<String> chunkIds() {
return Stream.of(
- "1_0_chunk0123456789abcdef",
- "12_34_chunk0123456789abcdef_ABC-xyz",
- "1_0_chunk0123456789abcdef~0~123",
- "1_690_chunkshort",
- "1/690/chunkshort",
- "12/12345/chunk550e8400-e29b-41d4-a716-446655440000",
- "1_12345_chunk550e8400-e29b-41d4-a716-446655440000",
- "1/690/chunkidentifier.with+other=characters~0~123",
- "1_690_chunkidentifier.with+other=characters",
- "1/690/chunk",
- "1_690_chunk");
+ "1_0_chunk_0123456789abcdef",
+ "12_34_chunk_0123456789abcdef_ABC-xyz",
+ "1_0_chunk_0123456789abcdef~0~123",
+ "1_690_chunk_short",
+ "1/690/chunk/short",
+ "12/12345/chunk/550e8400-e29b-41d4-a716-446655440000",
+ "1_12345_chunk_550e8400-e29b-41d4-a716-446655440000",
+ "1/690/chunk/identifier.with+other=characters~0~123",
+ "1_690_chunk_identifier.with+other=characters",
+ "1/690/chunk/",
+ "1_690_chunk_");
}
private static Stream<String> nonChunkIds() {
return Stream.of(
- "0_0_chunk0123456789abcdef",
- "1_-1_chunk0123456789abcdef",
- "0/690/chunkidentifier",
- "1/-1/chunkidentifier",
+ "0_0_chunk_0123456789abcdef",
+ "1_-1_chunk_0123456789abcdef",
+ "0/690/chunk/identifier",
+ "1/-1/chunk/identifier",
+ "1_690_chunkidentifier",
+ "1/690/chunkidentifier",
+ "1_690_chunk/identifier",
+ "1/690/chunk_identifier",
"1/690/ordinary-blob",
"1/690/hash_chunk_identifier",
- "1/690_chunkidentifier",
- "1_690/chunkidentifier",
+ "1/690_chunk_identifier",
+ "1_690/chunk/identifier",
"1_0_not-a-chunk0123456789abcdef",
"a_chunk0123456789abcdef",
"plain-blob-id");
diff --git a/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithmContract.java b/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithmContract.java
index 307f3d9169..a9b5d2acc2 100644
--- a/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithmContract.java
+++ b/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithmContract.java
@@ -150,7 +150,7 @@ public interface BloomFilterGCAlgorithmContract {
when(BLOB_REFERENCE_SOURCE.listReferencedBlobs()).thenReturn(Flux.empty());
BlobId normalBlob = GENERATION_AWARE_BLOB_ID_FACTORY.of(UUID.randomUUID().toString());
- BlobId chunkBlob = BLOB_ID_FACTORY.parse("1_1_chunk0123456789abcdef");
+ BlobId chunkBlob = BLOB_ID_FACTORY.parse("1_1_chunk_0123456789abcdef");
Mono.from(blobStoreDAO.save(DEFAULT_BUCKET, normalBlob, BlobStoreDAO.BytesBlob.of("normal".getBytes()))).block();
Mono.from(blobStoreDAO.save(DEFAULT_BUCKET, chunkBlob, BlobStoreDAO.BytesBlob.of("chunk".getBytes()))).block();
@@ -189,7 +189,7 @@ public interface BloomFilterGCAlgorithmContract {
when(BLOB_REFERENCE_SOURCE.listReferencedBlobs()).thenReturn(Flux.empty());
MinIOGenerationAwareBlobId.Factory minIOFactory = new MinIOGenerationAwareBlobId.Factory(
CLOCK, GENERATION_AWARE_BLOB_ID_CONFIGURATION, BLOB_ID_FACTORY);
- BlobId chunk = minIOFactory.parse("1/12/chunk" + UUID.randomUUID());
+ BlobId chunk = minIOFactory.parse("1/12/chunk/" + UUID.randomUUID());
BlobId ordinary = minIOFactory.parse("1/12/ordinary-blob");
Mono.from(store.save(DEFAULT_BUCKET, chunk, BlobStoreDAO.BytesBlob.of("chunk"))).block();
Mono.from(store.save(DEFAULT_BUCKET, ordinary, BlobStoreDAO.BytesBlob.of("ordinary"))).block();
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org