You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by be...@apache.org on 2022/01/26 07:16:10 UTC
[cassandra] branch trunk updated (ff4d63d -> 02dad36)
This is an automated email from the ASF dual-hosted git repository.
bereng pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git.
from ff4d63d Merge branch 'cassandra-4.0' into trunk
new 95257c0 Sstableverify unit test operate on SSTables
new 02dad36 Merge branch 'cassandra-4.0' into trunk
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.gitignore | 1 +
.../apache/cassandra/tools/StandaloneVerifier.java | 40 +++-
test/unit/org/apache/cassandra/db/VerifyTest.java | 40 ++--
.../apache/cassandra/tools/OfflineToolUtils.java | 3 -
.../tools/StandaloneVerifierOnSSTablesTest.java | 220 +++++++++++++++++++++
.../cassandra/tools/StandaloneVerifierTest.java | 30 ++-
6 files changed, 300 insertions(+), 34 deletions(-)
create mode 100644 test/unit/org/apache/cassandra/tools/StandaloneVerifierOnSSTablesTest.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra] 01/01: Merge branch 'cassandra-4.0' into trunk
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bereng pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 02dad368f1dbf38363cfe1a48f089cb2a7f24262
Merge: ff4d63d 95257c0
Author: Bereng <be...@gmail.com>
AuthorDate: Wed Jan 26 08:12:46 2022 +0100
Merge branch 'cassandra-4.0' into trunk
.gitignore | 1 +
.../apache/cassandra/tools/StandaloneVerifier.java | 40 +++-
test/unit/org/apache/cassandra/db/VerifyTest.java | 40 ++--
.../apache/cassandra/tools/OfflineToolUtils.java | 3 -
.../tools/StandaloneVerifierOnSSTablesTest.java | 220 +++++++++++++++++++++
.../cassandra/tools/StandaloneVerifierTest.java | 30 ++-
6 files changed, 300 insertions(+), 34 deletions(-)
diff --cc test/unit/org/apache/cassandra/db/VerifyTest.java
index 01baf97,b2f4344..571d5ef
--- a/test/unit/org/apache/cassandra/db/VerifyTest.java
+++ b/test/unit/org/apache/cassandra/db/VerifyTest.java
@@@ -18,8 -18,27 +18,25 @@@
*/
package org.apache.cassandra.db;
+ import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileInputStream;
+ import java.io.IOException;
+ import java.io.RandomAccessFile;
+ import java.net.UnknownHostException;
+ import java.nio.file.Files;
+ import java.util.ArrayList;
+ import java.util.Collections;
+ import java.util.List;
+ import java.util.UUID;
+ import java.util.concurrent.ExecutionException;
+ import java.util.zip.CRC32;
+ import java.util.zip.CheckedInputStream;
+
import com.google.common.base.Charsets;
+ import org.apache.commons.lang3.StringUtils;
+ import org.junit.BeforeClass;
+ import org.junit.Test;
+ import org.apache.cassandra.UpdateBuilder;
import org.apache.cassandra.Util;
import org.apache.cassandra.batchlog.Batch;
import org.apache.cassandra.batchlog.BatchlogManager;
@@@ -46,22 -63,7 +62,8 @@@ import org.apache.cassandra.schema.Comp
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
- import org.apache.commons.lang3.StringUtils;
- import org.junit.BeforeClass;
- import org.junit.Test;
-
- import java.io.*;
- import java.net.UnknownHostException;
- import java.nio.file.Files;
- import java.util.ArrayList;
- import java.util.Collections;
- import java.util.List;
- import java.util.UUID;
- import java.util.concurrent.ExecutionException;
- import java.util.zip.CRC32;
- import java.util.zip.CheckedInputStream;
+import org.apache.cassandra.io.util.File;
import static org.apache.cassandra.SchemaLoader.counterCFMD;
import static org.apache.cassandra.SchemaLoader.createKeyspace;
import static org.apache.cassandra.SchemaLoader.loadSchema;
diff --cc test/unit/org/apache/cassandra/tools/OfflineToolUtils.java
index 58c3836,7bd1143..7cabef5
--- a/test/unit/org/apache/cassandra/tools/OfflineToolUtils.java
+++ b/test/unit/org/apache/cassandra/tools/OfflineToolUtils.java
@@@ -31,12 -32,8 +31,9 @@@ import java.util.Set
import java.util.regex.Pattern;
import java.util.stream.Collectors;
+import org.apache.cassandra.io.util.File;
import org.apache.commons.io.FileUtils;
-
- import org.junit.AfterClass;
import org.junit.BeforeClass;
-
import org.slf4j.LoggerFactory;
import static org.apache.cassandra.utils.FBUtilities.preventIllegalAccessWarnings;
diff --cc test/unit/org/apache/cassandra/tools/StandaloneVerifierOnSSTablesTest.java
index 0000000,7df28d2..4a77a1c
mode 000000,100644..100644
--- a/test/unit/org/apache/cassandra/tools/StandaloneVerifierOnSSTablesTest.java
+++ b/test/unit/org/apache/cassandra/tools/StandaloneVerifierOnSSTablesTest.java
@@@ -1,0 -1,222 +1,220 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.cassandra.tools;
+
+ import java.io.File;
+ import java.io.RandomAccessFile;
+
+ import org.apache.commons.io.FileUtils;
+ import org.apache.commons.lang3.StringUtils;
+ import org.junit.AfterClass;
+ import org.junit.BeforeClass;
+ import org.junit.Test;
+
+ import org.apache.cassandra.SchemaLoader;
+ import org.apache.cassandra.UpdateBuilder;
+ import org.apache.cassandra.db.ColumnFamilyStore;
+ import org.apache.cassandra.db.Keyspace;
+ import org.apache.cassandra.db.PartitionPosition;
+ import org.apache.cassandra.db.compaction.CompactionManager;
+ import org.apache.cassandra.io.sstable.Component;
+ import org.apache.cassandra.io.sstable.format.SSTableReader;
+ import org.apache.cassandra.schema.KeyspaceParams;
+ import org.apache.cassandra.service.StorageService;
+ import org.apache.cassandra.tools.ToolRunner.ToolResult;
+ import org.apache.cassandra.utils.ByteBufferUtil;
+ import org.assertj.core.api.Assertions;
+
+ import static org.apache.cassandra.SchemaLoader.standardCFMD;
+ import static org.junit.Assert.assertEquals;
+
+ /**
+ * Class that tests tables for {@link StandaloneVerifier} by updating using {@link SchemaLoader}
+ * Similar in vein to other {@link SchemaLoader} type tests, as well as {@link StandaloneUpgraderOnSStablesTest}.
+ * Since the tool mainly exercises the {@link org.apache.cassandra.db.compaction.Verifier}, we elect to
+ * not run every conceivable option as many tests are already covered by {@link org.apache.cassandra.db.VerifyTest}.
+ *
+ * Note: the complete coverage is composed of:
+ * - {@link StandaloneVerifierOnSSTablesTest}
+ * - {@link StandaloneVerifierTest}
+ * - {@link org.apache.cassandra.db.VerifyTest}
+ */
+ public class StandaloneVerifierOnSSTablesTest extends OfflineToolUtils
+ {
+ @BeforeClass
+ public static void setup()
+ {
+ // since legacy tables test data uses ByteOrderedPartitioner that's what we need
+ // for the check version to work
+ System.setProperty("cassandra.partitioner", "org.apache.cassandra.dht.ByteOrderedPartitioner");
+ System.setProperty(Util.ALLOW_TOOL_REINIT_FOR_TEST, "true"); // Necessary for testing`
+ SchemaLoader.loadSchema();
+ StorageService.instance.initServer();
+ }
+
+ @AfterClass
+ public static void teardown() throws Exception
+ {
+ SchemaLoader.cleanupAndLeaveDirs();
+ System.clearProperty(Util.ALLOW_TOOL_REINIT_FOR_TEST);
+ }
+
+ @Test
+ public void testCheckVersionValidVersion() throws Exception
+ {
+ String keyspaceName = "StandaloneVerifierTestCheckVersionWorking";
+ String workingTable = "workingCheckTable";
+
+ createAndPopulateTable(keyspaceName, workingTable, x -> {});
+
+ ToolResult tool = ToolRunner.invokeClass(StandaloneVerifier.class, keyspaceName, workingTable, "-c");
+ assertEquals(0, tool.getExitCode());
- assertCorrectEnvPostTest();
+ tool.assertOnCleanExit();
+ }
+
+ @Test
+ public void testCheckVersionWithWrongVersion() throws Exception
+ {
+ String keyspace = "StandaloneVerifierTestWrongVersions";
+ String tableName = "legacy_ma_simple";
+
+ createAndPopulateTable(keyspace, tableName, cfs -> {
+ // let's just copy old version files from test data into the source dir
+ File testDataDir = new File("test/data/legacy-sstables/ma/legacy_tables/legacy_ma_simple");
- for (File cfsDir : cfs.getDirectories().getCFDirectories())
++ for (org.apache.cassandra.io.util.File cfsDir : cfs.getDirectories().getCFDirectories())
+ {
- FileUtils.copyDirectory(testDataDir, cfsDir);
++ FileUtils.copyDirectory(testDataDir, cfsDir.toJavaIOFile());
+ }
+ });
+
+ ToolResult tool = ToolRunner.invokeClass(StandaloneVerifier.class, keyspace, tableName, "-c");
+
+ assertEquals(1, tool.getExitCode());
+ Assertions.assertThat(tool.getStdout()).contains("is not the latest version, run upgradesstables");
+ }
+
+ @Test
+ public void testWorkingDataFile() throws Exception
+ {
+ String keyspaceName = "StandaloneVerifierTestWorkingDataKs";
+ String workingTable = "workingTable";
+
+ createAndPopulateTable(keyspaceName, workingTable, x -> {});
+
+ ToolResult tool = ToolRunner.invokeClass(StandaloneVerifier.class, keyspaceName, workingTable);
+ assertEquals(0, tool.getExitCode());
- assertCorrectEnvPostTest();
+ tool.assertOnCleanExit();
+ }
+
+ @Test
+ public void testCorruptStatsFile() throws Exception
+ {
+ String keyspaceName = "StandaloneVerifierTestCorruptStatsKs";
+ String corruptStatsTable = "corruptStatsTable";
+ createAndPopulateTable(keyspaceName, corruptStatsTable, cfs -> {
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ try (RandomAccessFile file = new RandomAccessFile(sstable.descriptor.filenameFor(Component.STATS), "rw"))
+ {
+ file.seek(0);
+ file.writeBytes(StringUtils.repeat('z', 2));
+ }
+ });
+
+ ToolResult tool = ToolRunner.invokeClass(StandaloneVerifier.class, keyspaceName, corruptStatsTable);
+
+ assertEquals(1, tool.getExitCode());
+ Assertions.assertThat(tool.getStderr()).contains("Error Loading", corruptStatsTable);
+ }
+
+ @Test
+ public void testCorruptDataFile() throws Exception
+ {
+ String keyspaceName = "StandaloneVerifierTestCorruptDataKs";
+ String corruptDataTable = "corruptDataTable";
+
+ createAndPopulateTable(keyspaceName, corruptDataTable, cfs -> {
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ long row0Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("0"), cfs.getPartitioner()), SSTableReader.Operator.EQ).position;
+ long row1Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("1"), cfs.getPartitioner()), SSTableReader.Operator.EQ).position;
+ long startPosition = Math.min(row0Start, row1Start);
+
+ try (RandomAccessFile file = new RandomAccessFile(sstable.getFilename(), "rw"))
+ {
+ file.seek(startPosition);
+ file.writeBytes(StringUtils.repeat('z', 2));
+ }
+ });
+
+ ToolResult tool = ToolRunner.invokeClass(StandaloneVerifier.class, keyspaceName, corruptDataTable);
+ assertEquals(1, tool.getExitCode());
+ Assertions.assertThat(tool.getStdout()).contains("Invalid SSTable", corruptDataTable);
+ }
+
+ /**
+ * Since we are testing a verifier, we'd like to corrupt files to verify code paths
+ * This function definition is used by {@link this#createAndPopulateTable}.
+ *
+ * CFS is the open ColumnFamilyStore for a given keyspace, table
+ */
+ @FunctionalInterface
+ private interface CorruptFunction
+ {
+ public void apply(ColumnFamilyStore cfs) throws Exception;
+ }
+
+ /**
+ * This function sets up the keyspace, and table schema for a standardCFMD table.
+ * <p>
+ * This will also populate the tableName with a few rows. After completion the
+ * server will be shutdown.
+ *
+ * @param keyspace the name of the keyspace in which the table should be created
+ * @param tableName new table name of the standard CFMD table
+ * @param corruptionFn function called to corrupt or change the contents on disk, is passed the Cfs of the table name.
+ * @throws Exception on error.
+ */
+ private static void createAndPopulateTable(String keyspace, String tableName, CorruptFunction corruptionFn) throws Exception
+ {
+ SchemaLoader.createKeyspace(keyspace,
+ KeyspaceParams.simple(1),
+ standardCFMD(keyspace, tableName));
+
+ CompactionManager.instance.disableAutoCompaction();
+
+ Keyspace k = Keyspace.open(keyspace);
+ ColumnFamilyStore cfs = k.getColumnFamilyStore(tableName);
+
+ populateTable(cfs, 2);
+
+ corruptionFn.apply(cfs);
+ }
+
+ private static void populateTable(ColumnFamilyStore cfs, int partitionsPerSSTable)
+ {
+ for (int i = 0; i < partitionsPerSSTable; i++)
+ {
+ UpdateBuilder.create(cfs.metadata(), String.valueOf(i))
+ .newRow("c1").add("val", "1")
+ .newRow("c2").add("val", "2")
+ .apply();
+ }
+
+ cfs.forceBlockingFlush();
+ }
+ }
diff --cc test/unit/org/apache/cassandra/tools/StandaloneVerifierTest.java
index 73890cd,9e6bb14..b76ceae
--- a/test/unit/org/apache/cassandra/tools/StandaloneVerifierTest.java
+++ b/test/unit/org/apache/cassandra/tools/StandaloneVerifierTest.java
@@@ -29,6 -29,12 +29,12 @@@ import org.hamcrest.CoreMatchers
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
+ /**
-* Note: the complete coverage is composed of:
++ * Note: the complete coverage is composed of:
+ * - {@link StandaloneVerifierOnSSTablesTest}
+ * - {@link StandaloneVerifierTest}
+ * - {@link org.apache.cassandra.db.VerifyTest}
+ */
public class StandaloneVerifierTest extends OfflineToolUtils
{
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org