You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ni...@apache.org on 2023/07/07 15:32:51 UTC
[ignite] branch master updated: IGNITE-19918 idle_verify stat for compact footer usage added (#10829)
This is an automated email from the ASF dual-hosted git repository.
nizhikov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new fd2a1f65a0b IGNITE-19918 idle_verify stat for compact footer usage added (#10829)
fd2a1f65a0b is described below
commit fd2a1f65a0be57d5dd38266e85010a0201437f3f
Author: Nikolay <ni...@apache.org>
AuthorDate: Fri Jul 7 18:32:44 2023 +0300
IGNITE-19918 idle_verify stat for compact footer usage added (#10829)
---
.../util/GridCommandHandlerClusterByClassTest.java | 51 ++++++++++++++
.../cache/VerifyBackupPartitionsDumpTask.java | 19 ++++++
.../IncrementalSnapshotVerificationTask.java | 7 +-
.../processors/cache/verify/IdleVerifyUtility.java | 40 ++++++++++-
.../cache/verify/PartitionHashRecordV2.java | 77 +++++++++++++++++++++-
5 files changed, 187 insertions(+), 7 deletions(-)
diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java
index 49f4a7159d7..13da56ba9ec 100644
--- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java
+++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java
@@ -59,12 +59,16 @@ import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.cache.affinity.AffinityFunction;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.client.IgniteClient;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.AtomicConfiguration;
+import org.apache.ignite.configuration.BinaryConfiguration;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.ClientConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteVersionUtils;
+import org.apache.ignite.internal.client.thin.TcpIgniteClient;
import org.apache.ignite.internal.commandline.ArgumentParser;
import org.apache.ignite.internal.commandline.CommandHandler;
import org.apache.ignite.internal.dto.IgniteDataTransferObject;
@@ -723,6 +727,7 @@ public class GridCommandHandlerClusterByClassTest extends GridCommandHandlerClus
assertContains(log, dumpWithZeros, "Partition: PartitionKeyV2 [grpId=1544803905, grpName=default, partId=0]");
assertContains(log, dumpWithZeros, "updateCntr=0, partitionState=OWNING, size=0, partHash=0");
assertContains(log, dumpWithZeros, "no conflicts have been found");
+ assertCompactFooterStat(dumpWithZeros, 0, 0, 0, keysCount);
assertSort(parts, dumpWithZeros);
}
@@ -741,11 +746,57 @@ public class GridCommandHandlerClusterByClassTest extends GridCommandHandlerClus
assertNotContains(log, dumpWithoutZeros, "updateCntr=0, partitionState=OWNING, size=0, partHash=0");
assertContains(log, dumpWithoutZeros, "no conflicts have been found");
+ assertCompactFooterStat(dumpWithoutZeros, 0, 0, 0, keysCount);
assertSort(keysCount, dumpWithoutZeros);
}
else
fail("Should be found both files");
+
+ for (int i = 0; i < keysCount / 2; i++)
+ ignite.cache(DEFAULT_CACHE_NAME).put(new TestClass(i, String.valueOf(i)), i);
+
+ assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", DEFAULT_CACHE_NAME));
+
+ fileNameMatcher = dumpFileNameMatcher();
+
+ assertTrue(fileNameMatcher.find());
+
+ String report = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1))));
+
+ assertCompactFooterStat(report, keysCount / 2, 0, keysCount / 2, keysCount);
+
+ ClientConfiguration cliCfg = new ClientConfiguration()
+ .setAddresses("127.0.0.1:10800")
+ .setAutoBinaryConfigurationEnabled(false)
+ .setBinaryConfiguration(new BinaryConfiguration().setCompactFooter(false));
+
+ try (IgniteClient cli = TcpIgniteClient.start(cliCfg)) {
+ for (int i = keysCount; i < keysCount * 3; i++)
+ cli.cache(DEFAULT_CACHE_NAME).put(new TestClass(i, String.valueOf(i)), i);
+ }
+
+ for (int i = 0; i < keysCount; i++)
+ ignite.cache(DEFAULT_CACHE_NAME).put(String.valueOf(i), i);
+
+ assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", DEFAULT_CACHE_NAME));
+
+ fileNameMatcher = dumpFileNameMatcher();
+
+ assertTrue(fileNameMatcher.find());
+
+ report = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1))));
+
+ assertCompactFooterStat(report, keysCount / 2, keysCount * 2, keysCount / 2 + keysCount * 2, keysCount * 2);
+ }
+
+ /** */
+ private static void assertCompactFooterStat(String report, long cf, long noCf, long binary, long regular) {
+ assertContains(log, report, "CompactFooter statistic for keys [" +
+ "compactFooter=" + cf + ", " +
+ "noCompactFooter=" + noCf + ", " +
+ "binary=" + binary + ", " +
+ "regular=" + regular + "]");
}
/**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsDumpTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsDumpTask.java
index fb06fbd9b98..7ac73ad9e88 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsDumpTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsDumpTask.java
@@ -240,14 +240,33 @@ public class VerifyBackupPartitionsDumpTask extends ComputeTaskAdapter<CacheIdle
if (!F.isEmpty(partitions)) {
writer.write("Cluster partitions:\n");
+ long cf = 0;
+ long noCf = 0;
+ long binary = 0;
+ long regular = 0;
+
for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : partitions.entrySet()) {
writer.write("Partition: " + entry.getKey() + "\n");
writer.write("Partition instances: " + entry.getValue() + "\n");
+
+ if (!entry.getValue().isEmpty()) {
+ PartitionHashRecordV2 rec = entry.getValue().get(0);
+
+ cf += rec.compactFooterKeys();
+ noCf += rec.noCompactFooterKeys();
+ binary += rec.binaryKeys();
+ regular += rec.regularKeys();
+ }
}
writer.write("\n\n-----------------------------------\n\n");
+ writer.write("CompactFooter statistic for keys [compactFooter=" + cf +
+ ", noCompactFooter=" + noCf +
+ ", binary=" + binary +
+ ", regular=" + regular + "]\n\n");
+
conflictRes.print(writer::write, true);
}
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotVerificationTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotVerificationTask.java
index c64cadb0f1e..2ce074d31e8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotVerificationTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotVerificationTask.java
@@ -378,7 +378,12 @@ public class IncrementalSnapshotVerificationTask extends AbstractSnapshotVerific
e.getValue().verHash,
null,
0,
- null)
+ null,
+ 0,
+ 0,
+ 0,
+ 0
+ )
));
if (log.isInfoEnabled()) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java
index eb3ff6f62aa..b93e1cf1394 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java
@@ -31,6 +31,7 @@ import java.util.function.BiConsumer;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.binary.BinaryObjectEx;
import org.apache.ignite.internal.management.cache.PartitionKeyV2;
import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.pagemem.PageIdUtils;
@@ -49,9 +50,11 @@ import org.apache.ignite.internal.util.typedef.internal.SB;
import org.apache.ignite.lang.IgniteInClosure;
import org.jetbrains.annotations.Nullable;
+import static org.apache.ignite.internal.binary.BinaryUtils.FLAG_COMPACT_FOOTER;
import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_AUX;
import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA;
import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX;
+import static org.apache.ignite.internal.processors.cache.CacheObject.TYPE_BINARY;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cacheGroupName;
/**
@@ -281,7 +284,11 @@ public class IdleVerifyUtility {
state == GridDhtPartitionState.MOVING ?
PartitionHashRecordV2.MOVING_PARTITION_SIZE : 0,
state == GridDhtPartitionState.MOVING ?
- PartitionHashRecordV2.PartitionState.MOVING : PartitionHashRecordV2.PartitionState.LOST);
+ PartitionHashRecordV2.PartitionState.MOVING : PartitionHashRecordV2.PartitionState.LOST,
+ 0,
+ 0,
+ 0,
+ 0);
}
if (state != GridDhtPartitionState.OWNING)
@@ -289,6 +296,10 @@ public class IdleVerifyUtility {
int partHash = 0;
int partVerHash = 0;
+ int cf = 0;
+ int noCf = 0;
+ int binary = 0;
+ int regular = 0;
while (it.hasNextX()) {
CacheDataRow row = it.nextX();
@@ -298,10 +309,33 @@ public class IdleVerifyUtility {
// Object context is not required since the valueBytes have been read directly from page.
partHash += Arrays.hashCode(row.value().valueBytes(null));
+
+ if (row.key().cacheObjectType() == TYPE_BINARY) {
+ binary++;
+
+ if (((BinaryObjectEx)row.key()).isFlagSet(FLAG_COMPACT_FOOTER))
+ cf++;
+ else
+ noCf++;
+ }
+ else
+ regular++;
}
- return new PartitionHashRecordV2(partKey, isPrimary, consId, partHash, partVerHash, updCntr,
- partSize, PartitionHashRecordV2.PartitionState.OWNING);
+ return new PartitionHashRecordV2(
+ partKey,
+ isPrimary,
+ consId,
+ partHash,
+ partVerHash,
+ updCntr,
+ partSize,
+ PartitionHashRecordV2.PartitionState.OWNING,
+ cf,
+ noCf,
+ binary,
+ regular
+ );
}
/**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecordV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecordV2.java
index 418517833dc..df7c98452d1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecordV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecordV2.java
@@ -20,6 +20,8 @@ import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.util.Objects;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.internal.binary.GridBinaryMarshaller;
import org.apache.ignite.internal.management.cache.PartitionKeyV2;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
@@ -68,6 +70,31 @@ public class PartitionHashRecordV2 extends VisorDataTransferObject {
/** Partition state. */
private PartitionState partitionState;
+ /**
+ * Count of keys with compact footer.
+ * @see BinaryConfiguration#isCompactFooter()
+ */
+ @GridToStringExclude
+ private int cfKeys;
+
+ /**
+ * Count of keys without compact footer.
+ * @see BinaryConfiguration#isCompactFooter()
+ */
+ @GridToStringExclude
+ private int noCfKeys;
+
+ /**
+ * Count of {@link org.apache.ignite.binary.BinaryObject} keys.
+ * @see GridBinaryMarshaller#BINARY_OBJ
+ */
+ @GridToStringExclude
+ private int binKeys;
+
+ /** Count of type supported by Ignite out of the box (numbers, strings, etc). */
+ @GridToStringExclude
+ private int regKeys;
+
/**
* @param partKey Partition key.
* @param isPrimary Is primary.
@@ -77,6 +104,10 @@ public class PartitionHashRecordV2 extends VisorDataTransferObject {
* @param updateCntr Update counter.
* @param size Size.
* @param partitionState Partition state.
+ * @param cfKeys Count of keys with compact footer.
+ * @param noCfKeys Count of keys without compact footer.
+ * @param binKeys Count of {@link org.apache.ignite.binary.BinaryObject} keys.
+ * @param regKeys Count of type supported by Ignite out of the box (numbers, strings, etc).
*/
public PartitionHashRecordV2(
PartitionKeyV2 partKey,
@@ -86,7 +117,11 @@ public class PartitionHashRecordV2 extends VisorDataTransferObject {
int partVerHash,
Object updateCntr,
long size,
- PartitionState partitionState
+ PartitionState partitionState,
+ int cfKeys,
+ int noCfKeys,
+ int binKeys,
+ int regKeys
) {
this.partKey = partKey;
this.isPrimary = isPrimary;
@@ -96,6 +131,10 @@ public class PartitionHashRecordV2 extends VisorDataTransferObject {
this.updateCntr = updateCntr;
this.size = size;
this.partitionState = partitionState;
+ this.cfKeys = cfKeys;
+ this.noCfKeys = noCfKeys;
+ this.binKeys = binKeys;
+ this.regKeys = regKeys;
}
/**
@@ -160,6 +199,26 @@ public class PartitionHashRecordV2 extends VisorDataTransferObject {
return partitionState;
}
+ /** */
+ public int compactFooterKeys() {
+ return cfKeys;
+ }
+
+ /** */
+ public int noCompactFooterKeys() {
+ return noCfKeys;
+ }
+
+ /** */
+ public int binaryKeys() {
+ return binKeys;
+ }
+
+ /** */
+ public int regularKeys() {
+ return regKeys;
+ }
+
/** {@inheritDoc} */
@Override protected void writeExternalData(ObjectOutput out) throws IOException {
out.writeObject(partKey);
@@ -170,6 +229,10 @@ public class PartitionHashRecordV2 extends VisorDataTransferObject {
out.writeObject(updateCntr);
out.writeLong(size);
U.writeEnum(out, partitionState);
+ out.writeInt(cfKeys);
+ out.writeInt(noCfKeys);
+ out.writeInt(binKeys);
+ out.writeInt(regKeys);
}
/** {@inheritDoc} */
@@ -187,6 +250,11 @@ public class PartitionHashRecordV2 extends VisorDataTransferObject {
partitionState = PartitionState.fromOrdinal(in.readByte());
else
partitionState = size == MOVING_PARTITION_SIZE ? PartitionState.MOVING : PartitionState.OWNING;
+
+ cfKeys = in.readInt();
+ noCfKeys = in.readInt();
+ binKeys = in.readInt();
+ regKeys = in.readInt();
}
/** {@inheritDoc} */
@@ -213,12 +281,15 @@ public class PartitionHashRecordV2 extends VisorDataTransferObject {
return partHash == v2.partHash && partVerHash == v2.partVerHash && Objects.equals(updateCntr, v2.updateCntr) &&
size == v2.size && partKey.equals(v2.partKey) && consistentId.equals(v2.consistentId) &&
- partitionState == v2.partitionState;
+ partitionState == v2.partitionState &&
+ cfKeys == v2.cfKeys && noCfKeys == v2.noCfKeys &&
+ binKeys == v2.binKeys && regKeys == v2.regKeys;
}
/** {@inheritDoc} */
@Override public int hashCode() {
- return Objects.hash(partKey, consistentId, partHash, partVerHash, updateCntr, size, partitionState);
+ return Objects.hash(partKey, consistentId, partHash, partVerHash, updateCntr, size, partitionState,
+ cfKeys, noCfKeys, binKeys, regKeys);
}
/** **/