You are viewing a plain text version of this content. The canonical link for it is here.
Posted to pr@cassandra.apache.org by GitBox <gi...@apache.org> on 2021/01/14 21:59:26 UTC
[GitHub] [cassandra] maedhroz commented on a change in pull request #801: CASSANDRA-16180 Coordination tests
maedhroz commented on a change in pull request #801:
URL: https://github.com/apache/cassandra/pull/801#discussion_r557725616
##########
File path: test/distributed/org/apache/cassandra/distributed/test/PutGetTest.java
##########
@@ -0,0 +1,306 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.collect.Iterators;
+import org.apache.commons.lang3.ArrayUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ICoordinator;
+import org.apache.cassandra.io.compress.DeflateCompressor;
+import org.apache.cassandra.io.compress.LZ4Compressor;
+import org.apache.cassandra.io.compress.NoopCompressor;
+import org.apache.cassandra.io.compress.SnappyCompressor;
+import org.apache.cassandra.io.compress.ZstdCompressor;
+
+import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM;
+import static org.apache.cassandra.distributed.shared.AssertUtils.assertRow;
+import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
+import static org.apache.cassandra.distributed.shared.AssertUtils.row;
+
+/**
+ * Simple put/get tests using different types of query, paticularly when the data is spread across memory and multiple
+ * sstables and using different compressors. All available compressors are tested. Both ascending and descending
+ * clustering orders are tested. The read queries are run using every node as a coordinator, with and without paging.
+ */
+@RunWith(Parameterized.class)
+public class PutGetTest extends TestBaseImpl
+{
+ private static final int NUM_NODES = 4;
+ private static final int REPLICATION_FACTOR = 3;
+ private static final String CREATE_TABLE = "CREATE TABLE %s(k int, c int, v int, PRIMARY KEY (k, c)) " +
+ "WITH CLUSTERING ORDER BY (c %s) " +
+ "AND COMPRESSION = { 'class': '%s' } " +
+ "AND READ_REPAIR = 'none'";
+ private static final String[] COMPRESSORS = new String[]{ NoopCompressor.class.getSimpleName(),
+ LZ4Compressor.class.getSimpleName(),
+ DeflateCompressor.class.getSimpleName(),
+ SnappyCompressor.class.getSimpleName(),
+ ZstdCompressor.class.getSimpleName() };
+
+ private static final AtomicInteger seq = new AtomicInteger();
+
+ /**
+ * The sstable compressor to be used.
+ */
+ @Parameterized.Parameter
+ public String compressor;
+
+ /**
+ * Whether the clustering order is reverse.
+ */
+ @Parameterized.Parameter(1)
+ public boolean reverse;
+
+ private String tableName;
+
+ @Parameterized.Parameters(name = "{index}: compressor={0} reverse={1}")
+ public static Collection<Object[]> data()
+ {
+ List<Object[]> result = new ArrayList<>();
+ for (String compressor : COMPRESSORS)
+ for (boolean reverse : BOOLEANS)
+ result.add(new Object[]{ compressor, reverse });
+ return result;
+ }
+
+ private static Cluster cluster;
+
+ @BeforeClass
+ public static void setupCluster() throws IOException
+ {
+ cluster = init(Cluster.build(NUM_NODES).start(), REPLICATION_FACTOR);
+ }
+
+ @AfterClass
+ public static void teardownCluster()
+ {
+ if (cluster != null)
+ cluster.close();
+ }
+
+ @Before
+ public void before()
+ {
+ // create the table
+ tableName = String.format("%s.t_%d", KEYSPACE, seq.getAndIncrement());
+ cluster.schemaChange(String.format(CREATE_TABLE, tableName, reverse ? "DESC" : "ASC", compressor));
+ }
+
+ @After
+ public void after()
+ {
+ cluster.schemaChange(format("DROP TABLE %s"));
+ }
+
+ /**
+ * Simple put/get on a single partition with a few rows, reading with a single partition query.
+ * <p>
+ * Migrated from Python dtests putget_test.py:TestPutGet.test_putget[_snappy|_deflate]().
+ */
+ @Test
+ public void testPartitionQuery()
+ {
+ int numRows = 10;
+
+ writeRows(1, numRows);
+
+ Object[][] rows = readRows("SELECT * FROM %s WHERE k=?", 0);
+ Assert.assertEquals(numRows, rows.length);
+ for (int c = 0; c < numRows; c++)
+ {
+ validateRow(rows[c], numRows, 0, c);
+ }
+ }
+
+ /**
+ * Simple put/get on multiple partitions with multiple rows, reading with a range query.
+ * <p>
+ * Migrated from Python dtests putget_test.py:TestPutGet.test_rangeputget().
+ */
+ @Test
+ public void testRangeQuery()
+ {
+ int numPartitions = 10;
+ int numRows = 10;
+
+ writeRows(numPartitions, numRows);
+
+ Object[][] rows = readRows("SELECT * FROM %s");
+ Assert.assertEquals(numPartitions * numRows, rows.length);
+ for (int k = 0; k < numPartitions; k++)
+ {
+ for (int c = 0; c < numRows; c++)
+ {
+ Object[] row = rows[k * numRows + c];
+ validateRow(row, numRows, k, c);
+ }
+ }
+ }
+
+ /**
+ * Simple put/get on a single partition with multiple rows, reading with slice queries.
+ * <p>
+ * Migrated from Python dtests putget_test.py:TestPutGet.test_wide_row().
+ */
+ @Test
+ public void testSliceQuery()
+ {
+ int numRows = 100;
+
+ writeRows(1, numRows);
+
+ String query = "SELECT * FROM %s WHERE k=? AND c>=? AND c<?";
+ for (int sliceSize : Arrays.asList(10, 20, 100))
+ {
+ for (int c = 0; c < numRows; c = c + sliceSize)
+ {
+ Object[][] rows = readRows(query, 0, c, c + sliceSize);
+ Assert.assertEquals(sliceSize, rows.length);
+
+ for (int i = 0; i < sliceSize; i++)
+ {
+ Object[] row = rows[i];
+ validateRow(row, numRows, 0, c + i);
+ }
+ }
+ }
+ }
+
+ /**
+ * Simple put/get on multiple partitions with multiple rows, reading with IN queries.
+ */
+ @Test
+ public void testInQuery()
+ {
+ int numPartitions = 10;
+ int numRows = 10;
+
+ writeRows(numPartitions, numRows);
+
+ String query = "SELECT * FROM %s WHERE k IN (?, ?)";
+ for (int k = 0; k < numPartitions; k += 2)
+ {
+ Object[][] rows = readRows(query, k, k + 1);
+ Assert.assertEquals(numRows * 2, rows.length);
+
+ for (int i = 0; i < 2; i++)
+ {
+ for (int c = 0; c < numRows; c++)
+ {
+ Object[] row = rows[i * numRows + c];
+ validateRow(row, numRows, k + i, c);
+ }
+ }
+ }
+ }
+
+ /**
+ * Writes {@code numPartitions} with {@code numRows} each, with overrides in different sstables and memtables.
+ */
+ private void writeRows(int numPartitions, int numRows)
+ {
+ String update = format("UPDATE %s SET v=? WHERE k=? AND c=?");
+ ICoordinator coordinator = cluster.coordinator(1);
+ for (int k = 0; k < numPartitions; k++)
+ {
+ // insert all the partition rows in a single sstable
+ for (int c = 0; c < numRows; c++)
+ coordinator.execute(update, QUORUM, c, k, c);
+ cluster.forEach(i -> i.flush(KEYSPACE));
+
+ // override some rows in a second sstable
+ for (int c = 0; c < numRows; c += 2)
+ coordinator.execute(update, QUORUM, c + numRows, k, c);
+ cluster.forEach(i -> i.flush(KEYSPACE));
+
+ // override some rows only in memtable
+ for (int c = 0; c < numRows; c += 5)
+ coordinator.execute(update, QUORUM, c + numRows * 2, k, c);
+ }
+ }
+
+ /**
+ * Runs the specified query in all coordinators, with and without paging.
+ */
+ private Object[][] readRows(String query, Object... boundValues)
+ {
+ query = format(query);
+ Object[][] rows = null;
+
+ // verify that all coordinators return the same results for the query, regardless of paging
+ for (ICoordinator coordinator : cluster.coordinators())
+ {
+ for (boolean paging : BOOLEANS)
+ {
+ Object[][] readRows = paging
+ ? Iterators.toArray(coordinator.executeWithPaging(query, QUORUM, 1, boundValues),
+ Object[].class)
+ : coordinator.execute(query, QUORUM, boundValues);
+ if (rows == null)
+ rows = readRows;
+ else
+ assertRows(rows, readRows);
+ }
+ }
+ Assert.assertNotNull(rows);
+
+ // undo the clustering reverse sorting to ease validation
+ if (reverse)
+ ArrayUtils.reverse(rows);
+
+ // sort by partition key to ease validation
+ Arrays.sort(rows, Comparator.comparing(row -> (int) row[0]));
+
+ return rows;
+ }
+
+ private void validateRow(Object[] row, int numRows, int k, int c)
+ {
+ Assert.assertNotNull(row);
+
+ if (c % 5 == 0)
+ assertRow(row, row(k, c, c + numRows * 2));
+ else if (c % 2 == 0)
+ assertRow(row, row(k, c, c + numRows));
+ else
+ assertRow(row, row(k, c, c));
+ }
+
+ private String format(String query)
Review comment:
nit: Perhaps we could call this `withTable()` or something similar?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: pr-unsubscribe@cassandra.apache.org
For additional commands, e-mail: pr-help@cassandra.apache.org