You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by mc...@apache.org on 2020/10/09 14:02:34 UTC
[cassandra] branch trunk updated: sstablesplit tool unit testing
This is an automated email from the ASF dual-hosted git repository.
mck pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 4564e10 sstablesplit tool unit testing
4564e10 is described below
commit 4564e102684dc5f66ec73de1fc836f97a1fa33c9
Author: Bereng <be...@gmail.com>
AuthorDate: Thu Oct 1 10:56:17 2020 +0200
sstablesplit tool unit testing
patch by Berenguer Blasi; reviewed by Yifan Cai, Mick Semb Wever for CASSANDRA-16012
---
.../apache/cassandra/tools/StandaloneSplitter.java | 6 +-
src/java/org/apache/cassandra/tools/Util.java | 1 +
.../tools/StandaloneSplitterWithCQLTesterTest.java | 188 +++++++++++++++++++++
3 files changed, 194 insertions(+), 1 deletion(-)
diff --git a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java b/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
index 7a60b6f..e15e5bc 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
@@ -26,6 +26,7 @@ import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.commons.cli.*;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.Directories;
@@ -51,7 +52,10 @@ public class StandaloneSplitter
public static void main(String args[])
{
Options options = Options.parseArgs(args);
- Util.initDatabaseDescriptor();
+ if (Boolean.getBoolean(Util.ALLOW_TOOL_REINIT_FOR_TEST))
+ DatabaseDescriptor.toolInitialization(false); //Necessary for testing
+ else
+ Util.initDatabaseDescriptor();
try
{
diff --git a/src/java/org/apache/cassandra/tools/Util.java b/src/java/org/apache/cassandra/tools/Util.java
index db664aa..7411858 100644
--- a/src/java/org/apache/cassandra/tools/Util.java
+++ b/src/java/org/apache/cassandra/tools/Util.java
@@ -54,6 +54,7 @@ import com.google.common.collect.Lists;
@SuppressWarnings("serial")
public final class Util
{
+ static final String ALLOW_TOOL_REINIT_FOR_TEST = Util.class.getName() + "ALLOW_TOOL_REINIT_FOR_TEST"; // Necessary for testing
static final String RESET = "\u001B[0m";
static final String BLUE = "\u001B[34m";
static final String CYAN = "\u001B[36m";
diff --git a/test/unit/org/apache/cassandra/tools/StandaloneSplitterWithCQLTesterTest.java b/test/unit/org/apache/cassandra/tools/StandaloneSplitterWithCQLTesterTest.java
new file mode 100644
index 0000000..9785d84
--- /dev/null
+++ b/test/unit/org/apache/cassandra/tools/StandaloneSplitterWithCQLTesterTest.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.google.common.io.Files;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.apache.cassandra.OrderedJUnit4ClassRunner;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.tools.ToolRunner.ToolResult;
+import org.assertj.core.api.Assertions;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(OrderedJUnit4ClassRunner.class)
+public class StandaloneSplitterWithCQLTesterTest extends CQLTester
+{
+ private static String sstableFileName;
+ private static File sstablesDir;
+ private static File sstablesBackupDir;
+ private static List<File> origSstables;
+
+ // CQLTester post test method cleanup needs to be avoided by overriding as it'd clean all sstables, env, etc.
+ @Override
+ public void afterTest() throws Throwable
+ {
+ }
+
+ @Test
+ public void setupEnv() throws Throwable
+ {
+ // Stop the server after setup as we're going to be changing things under it's feet
+ setupTestSstables();
+ tearDownClass();
+ }
+
+ @Test
+ public void testMinFileSizeCheck() throws Throwable
+ {
+ restoreOrigSstables();
+ ToolResult tool = ToolRunner.invokeClass(StandaloneSplitter.class, sstableFileName);
+ Assertions.assertThat(tool.getStdout()).contains("is less than the split size");
+ assertTrue(tool.getCleanedStderr(), tool.getCleanedStderr().isEmpty());
+ assertEquals(0, tool.getExitCode());
+ }
+
+ @Test
+ public void testSplittingSSTable() throws Throwable
+ {
+ restoreOrigSstables();
+
+ ToolResult tool = ToolRunner.invokeClass(StandaloneSplitter.class, "-s", "1", sstableFileName);
+ List<File> splitFiles = Arrays.asList(sstablesDir.listFiles());
+ splitFiles.stream().forEach(f -> {
+ if (f.getName().endsWith("Data.db") && !origSstables.contains(f))
+ assertTrue(f.getName() + " is way bigger than 1MB: [" + f.length() + "] bytes",
+ f.length() <= 1024 * 1024 * 1.2); //give a 20% margin on size check
+ });
+ assertTrue(origSstables.size() < splitFiles.size());
+ Assertions.assertThat(tool.getStdout()).contains("sstables snapshotted into");
+ assertTrue(tool.getCleanedStderr(), tool.getCleanedStderr().isEmpty());
+ assertEquals(0, tool.getExitCode());
+ }
+
+ @Test
+ public void testSplittingMultipleSSTables() throws Throwable
+ {
+ restoreOrigSstables();
+ ArrayList<String> args = new ArrayList<>(Arrays.asList("-s", "1"));
+
+ args.addAll(Arrays.asList(sstablesDir.listFiles())
+ .stream()
+ .map(f -> f.getAbsolutePath())
+ .collect(Collectors.toList()));
+
+ ToolResult tool = ToolRunner.invokeClass(StandaloneSplitter.class, args.toArray(new String[args.size()]));
+ List<File> splitFiles = Arrays.asList(sstablesDir.listFiles());
+ splitFiles.stream().forEach(f -> {
+ if (f.getName().endsWith("Data.db") && !origSstables.contains(f))
+ assertTrue(f.getName() + " is way bigger than 1MB: [" + f.length() + "] bytes",
+ f.length() <= 1024 * 1024 * 1.2); //give a 20% margin on size check
+ });
+ assertTrue(origSstables.size() < splitFiles.size());
+ assertTrue(tool.getCleanedStderr(), tool.getCleanedStderr().isEmpty());
+ assertEquals(0, tool.getExitCode());
+ }
+
+ @Test
+ public void testNoSnapshotOption() throws Throwable
+ {
+ restoreOrigSstables();
+ ToolResult tool = ToolRunner.invokeClass(StandaloneSplitter.class, "-s", "1", "--no-snapshot", sstableFileName);
+ assertTrue(origSstables.size() < Arrays.asList(sstablesDir.listFiles()).size());
+ assertTrue(tool.getStdout(), tool.getStdout().isEmpty());
+ assertTrue(tool.getCleanedStderr(), tool.getCleanedStderr().isEmpty());
+ assertEquals(0, tool.getExitCode());
+ }
+
+ @Test
+ public void cleanEnv() throws Throwable
+ {
+ super.afterTest();
+ System.clearProperty(Util.ALLOW_TOOL_REINIT_FOR_TEST);
+ }
+
+ private void setupTestSstables() throws Throwable
+ {
+ createTable("CREATE TABLE %s (id text primary key, val text)");
+ for (int i = 0; i < 100000; i++)
+ executeFormattedQuery(formatQuery("INSERT INTO %s (id, val) VALUES (?, ?)"), "mockData" + i, "mockData" + i);
+
+ ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+ cfs.forceBlockingFlush();
+
+ Set<SSTableReader> sstables = cfs.getLiveSSTables();
+ sstableFileName = sstables.iterator().next().getFilename();
+ assertTrue("Generated sstable must be at least 1MB", (new File(sstableFileName)).length() > 1024*1024);
+ sstablesDir = new File(sstableFileName).getParentFile();
+ sstablesBackupDir = new File(sstablesDir.getAbsolutePath() + "/testbackup");
+ sstablesBackupDir.mkdir();
+ origSstables = Arrays.asList(sstablesDir.listFiles());
+
+ // Back up orig sstables
+ origSstables.stream().forEach(f -> {
+ if (f.isFile())
+ try
+ {
+ Files.copy(f, new File(sstablesBackupDir.getAbsolutePath() + "/" + f.getName()));
+ }
+ catch(IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ });
+
+ System.setProperty(Util.ALLOW_TOOL_REINIT_FOR_TEST, "true"); // Necessary for testing
+ }
+
+ private void restoreOrigSstables()
+ {
+ Arrays.asList(sstablesDir.listFiles()).stream().forEach(f -> {
+ if (f.isFile())
+ f.delete();
+ });
+ Arrays.asList(sstablesBackupDir.listFiles()).stream().forEach(f -> {
+ if (f.isFile())
+ try
+ {
+ Files.copy(f, new File(sstablesDir.getAbsolutePath() + "/" + f.getName()));
+ }
+ catch(IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ });
+
+ SSTableReader.resetTidying();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org