You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2025/11/06 17:28:29 UTC
(asterixdb) 12/25: [NO ISSUE][*DB][STO] Increased log brevity in cloud storage
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 44adbba68d676ee2ed0d523a73138f0759af4c9e
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sun Oct 12 09:37:04 2025 -0400
[NO ISSUE][*DB][STO] Increased log brevity in cloud storage
Ext-ref: MB-68946
Change-Id: I37332782d583d5c9daaf8b58dad7c4a1a2992bd6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20483
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../asterix/cloud/AbstractCloudIOManager.java | 10 +++---
.../cloud/lazy/accessor/InitialCloudAccessor.java | 5 ++-
.../lazy/accessor/ReplaceableCloudAccessor.java | 5 +--
.../lazy/accessor/SelectiveCloudAccessor.java | 4 +--
.../apache/asterix/cloud/util/CloudFileUtil.java | 1 -
.../apache/asterix/common/utils/Partitions.java | 38 ++++++++++++++++++++--
6 files changed, 47 insertions(+), 16 deletions(-)
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
index 526af2df1c..7712f08653 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
@@ -31,7 +31,6 @@ import java.nio.file.FileStore;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Predicate;
@@ -50,6 +49,7 @@ import org.apache.asterix.common.cloud.IPartitionBootstrapper;
import org.apache.asterix.common.config.CloudProperties;
import org.apache.asterix.common.metadata.MetadataConstants;
import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
+import org.apache.asterix.common.utils.Partitions;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
@@ -82,12 +82,11 @@ public abstract class AbstractCloudIOManager extends IOManager implements IParti
protected final ICloudGuardian guardian;
protected final IWriteBufferProvider writeBufferProvider;
protected final String bucket;
- protected final Set<Integer> partitions;
+ protected final Partitions partitions;
protected final List<FileReference> partitionPaths;
protected final IOManager localIoManager;
protected final INamespacePathResolver nsPathResolver;
private final List<FileStore> drivePaths;
- private final String storageScheme;
public AbstractCloudIOManager(IOManager ioManager, CloudProperties cloudProperties,
INamespacePathResolver nsPathResolver, ICloudGuardian guardian) throws HyracksDataException {
@@ -99,11 +98,10 @@ public abstract class AbstractCloudIOManager extends IOManager implements IParti
this.guardian = guardian;
int numOfThreads = getIODevices().size() * getIOParallelism();
writeBufferProvider = new WriteBufferProvider(numOfThreads, cloudClient.getWriteBufferSize());
- partitions = new HashSet<>();
+ partitions = new Partitions();
partitionPaths = new ArrayList<>();
this.localIoManager = ioManager;
drivePaths = PhysicalDrive.getDrivePaths(ioDevices);
- storageScheme = cloudProperties.getStorageScheme();
}
/*
@@ -179,7 +177,7 @@ public abstract class AbstractCloudIOManager extends IOManager implements IParti
}
}
} else {
- LOGGER.info("Cleaning node partitions...");
+ LOGGER.info("Cleaning node partitions {}...", partitions);
for (FileReference partitionPath : partitionPaths) {
CloudFileUtil.cleanDirectoryFiles(localIoManager, cloudFiles, partitionPath);
}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java
index 8c3321ad86..5070d40bce 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java
@@ -18,10 +18,9 @@
*/
package org.apache.asterix.cloud.lazy.accessor;
-import java.util.Collections;
-
import org.apache.asterix.cloud.clients.ICloudClient;
import org.apache.asterix.cloud.lazy.NoOpParallelCacher;
+import org.apache.asterix.common.utils.Partitions;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.control.nc.io.IOManager;
@@ -35,7 +34,7 @@ public class InitialCloudAccessor extends ReplaceableCloudAccessor {
};
public InitialCloudAccessor(ICloudClient cloudClient, String bucket, IOManager localIoManager) {
- super(cloudClient, bucket, localIoManager, Collections.emptySet(), NO_OP_REPLACER, NoOpParallelCacher.INSTANCE);
+ super(cloudClient, bucket, localIoManager, Partitions.empty(), NO_OP_REPLACER, NoOpParallelCacher.INSTANCE);
}
@Override
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java
index 91046a1a06..8dbf605608 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java
@@ -28,6 +28,7 @@ import org.apache.asterix.cloud.bulk.IBulkOperationCallBack;
import org.apache.asterix.cloud.clients.CloudFile;
import org.apache.asterix.cloud.clients.ICloudClient;
import org.apache.asterix.cloud.lazy.IParallelCacher;
+import org.apache.asterix.common.utils.Partitions;
import org.apache.asterix.common.utils.StorageConstants;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -42,13 +43,13 @@ import org.apache.logging.log4j.Logger;
*/
public class ReplaceableCloudAccessor extends AbstractLazyAccessor {
private static final Logger LOGGER = LogManager.getLogger();
- private final Set<Integer> partitions;
+ private final Partitions partitions;
private final ILazyAccessorReplacer replacer;
protected final IParallelCacher cacher;
private final IBulkOperationCallBack deleteCallBack;
public ReplaceableCloudAccessor(ICloudClient cloudClient, String bucket, IOManager localIoManager,
- Set<Integer> partitions, ILazyAccessorReplacer replacer, IParallelCacher cacher) {
+ Partitions partitions, ILazyAccessorReplacer replacer, IParallelCacher cacher) {
super(cloudClient, bucket, localIoManager);
this.partitions = partitions;
this.replacer = replacer;
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java
index 56eaeeb08b..68c0556fa3 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java
@@ -21,12 +21,12 @@ package org.apache.asterix.cloud.lazy.accessor;
import static org.apache.asterix.cloud.util.CloudFileUtil.DATA_FILTER;
import java.util.Collection;
-import java.util.Set;
import org.apache.asterix.cloud.UncachedFileReference;
import org.apache.asterix.cloud.clients.ICloudClient;
import org.apache.asterix.cloud.lazy.IParallelCacher;
import org.apache.asterix.cloud.lazy.filesystem.IHolePuncher;
+import org.apache.asterix.common.utils.Partitions;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IFileHandle;
@@ -37,7 +37,7 @@ public class SelectiveCloudAccessor extends ReplaceableCloudAccessor {
private final IHolePuncher puncher;
public SelectiveCloudAccessor(ICloudClient cloudClient, String bucket, IOManager localIoManager,
- Set<Integer> partitions, IHolePuncher puncher, IParallelCacher cacher) {
+ Partitions partitions, IHolePuncher puncher, IParallelCacher cacher) {
super(cloudClient, bucket, localIoManager, partitions, InitialCloudAccessor.NO_OP_REPLACER, cacher);
this.puncher = puncher;
}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
index 7befacf027..fd37e61c04 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
@@ -49,7 +49,6 @@ public class CloudFileUtil {
// First get the set of local files
Set<FileReference> localFiles = ioManager.list(partitionPath);
Iterator<FileReference> localFilesIter = localFiles.iterator();
- LOGGER.info("Cleaning partition {}.", partitionPath.getRelativePath());
// Reconcile local files and cloud files
while (localFilesIter.hasNext()) {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java
index 45aaa49150..cc93b916cc 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java
@@ -20,9 +20,13 @@ package org.apache.asterix.common.utils;
import java.io.Serial;
import java.io.Serializable;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.MutableBoolean;
import it.unimi.dsi.fastutil.ints.IntIterator;
import it.unimi.dsi.fastutil.ints.IntSortedSet;
+import it.unimi.dsi.fastutil.ints.IntSortedSets;
/**
* A specialized bit set for storing partition IDs.
@@ -36,21 +40,29 @@ public class Partitions implements Serializable {
private static final long serialVersionUID = 1L;
private static final int MAX_PARTITION_ID = Integer.MAX_VALUE - 1;
private static final int MIN_PARTITION_ID = -1;
+ private static final Partitions EMPTY = new Partitions(IntSortedSets.EMPTY_SET);
private static final short MINUS_ONE = (short) -1;
private final IntSortedSet delegate;
public Partitions() {
- delegate = new IntSortedBitSet();
+ this(new IntSortedBitSet());
}
public Partitions(int initialMaxValue) {
- delegate = new IntSortedBitSet(initialMaxValue + 1);
+ this(new IntSortedBitSet(initialMaxValue + 1));
}
public Partitions(IntSortedSet delegate) {
this.delegate = delegate;
}
+ /**
+ * Returns an unmodifiable empty Partitions instance.
+ */
+ public static Partitions empty() {
+ return EMPTY;
+ }
+
/**
* Adds a partition ID to the set. Partition ID must be in the range [-1, 32766].
* @param k the partition ID to add
@@ -72,6 +84,27 @@ public class Partitions implements Serializable {
delegate.clear();
}
+ public boolean addAll(Set<Integer> activePartitions) {
+ MutableBoolean retval = new MutableBoolean();
+ activePartitions.forEach(p -> {
+ if (add(p)) {
+ retval.setTrue();
+ }
+ });
+ return retval.booleanValue();
+ }
+
+ public int size() {
+ return delegate.size();
+ }
+
+ public boolean contains(int partitionNum) {
+ if (partitionNum > MAX_PARTITION_ID || partitionNum < MIN_PARTITION_ID) {
+ return false;
+ }
+ return delegate.contains((short) (partitionNum + 1));
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -107,4 +140,5 @@ public class Partitions implements Serializable {
builder.append(']');
return builder.toString();
}
+
}