You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by mc...@apache.org on 2021/04/28 08:49:15 UTC
[cassandra] branch trunk updated: Fix compression test runs
This is an automated email from the ASF dual-hosted git repository.
mck pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push:
new 10a1d65 Fix compression test runs
10a1d65 is described below
commit 10a1d65eb09a93aee32948b46b4f1a0fbc2defe0
Author: jacek-lewandowski <ja...@datastax.com>
AuthorDate: Fri Apr 23 09:15:22 2021 +0200
Fix compression test runs
Ensure the specified commitlog_compression_xxx.yaml file is appended to the used cassandra.compressed.yaml, according to the selected algorithm as defined by `test.compression.algo` the property. By default we use the default LZ4. An unknown compression algo with fail.
patch by Jacek Lewandowski; reviewed by Mick Semb Wever for CASSANDRA-16575
---
build.xml | 5 ++--
test/unit/org/apache/cassandra/SchemaLoader.java | 34 ++++++++++++++++++------
2 files changed, 29 insertions(+), 10 deletions(-)
diff --git a/build.xml b/build.xml
index 347ab6e..0a40b2f 100644
--- a/build.xml
+++ b/build.xml
@@ -61,7 +61,7 @@
<property name="test.memory.src" value="${test.dir}/memory"/>
<property name="test.microbench.src" value="${test.dir}/microbench"/>
<property name="test.distributed.src" value="${test.dir}/distributed"/>
- <property name="test.compression_algo" value="LZ4"/>
+ <property name="test.compression.algo" value="LZ4"/>
<property name="test.driver.connection_timeout_ms" value="5000"/>
<property name="test.driver.read_timeout_ms" value="12000"/>
<property name="dist.dir" value="${build.dir}/dist"/>
@@ -1430,13 +1430,14 @@
<property name="compressed_yaml" value="${build.test.dir}/cassandra.compressed.yaml"/>
<concat destfile="${compressed_yaml}">
<fileset file="${test.conf}/cassandra.yaml"/>
- <fileset file="${test.conf}/commitlog_compression.yaml"/>
+ <fileset file="${test.conf}/commitlog_compression_${test.compression.algo}.yaml"/>
</concat>
<testmacrohelper inputdir="${test.unit.src}" filelist="@{test.file.list}"
exclude="**/*.java" timeout="${test.timeout}" testtag="compression">
<jvmarg value="-Dlegacy-sstable-root=${test.data}/legacy-sstables"/>
<jvmarg value="-Dinvalid-legacy-sstable-root=${test.data}/invalid-legacy-sstables"/>
<jvmarg value="-Dcassandra.test.compression=true"/>
+ <jvmarg value="-Dcassandra.test.compression.algo=${test.compression.algo}"/>
<jvmarg value="-Dcassandra.ring_delay_ms=1000"/>
<jvmarg value="-Dcassandra.tolerate_sstable_size=true"/>
<jvmarg value="-Dcassandra.config=file:///${compressed_yaml}"/>
diff --git a/test/unit/org/apache/cassandra/SchemaLoader.java b/test/unit/org/apache/cassandra/SchemaLoader.java
index 001e155..1f46562 100644
--- a/test/unit/org/apache/cassandra/SchemaLoader.java
+++ b/test/unit/org/apache/cassandra/SchemaLoader.java
@@ -17,7 +17,6 @@
*/
package org.apache.cassandra;
-import java.io.File;
import java.io.IOException;
import java.util.*;
@@ -28,12 +27,10 @@ import org.apache.cassandra.auth.IAuthorizer;
import org.apache.cassandra.auth.INetworkAuthorizer;
import org.apache.cassandra.auth.IRoleManager;
import org.apache.cassandra.config.*;
-import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.ColumnIdentifier;
import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
import org.apache.cassandra.cql3.statements.schema.IndexTarget;
import org.apache.cassandra.db.RowUpdateBuilder;
-import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.db.marshal.*;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.exceptions.ConfigurationException;
@@ -41,7 +38,6 @@ import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.index.StubIndex;
import org.apache.cassandra.index.sasi.SASIIndex;
import org.apache.cassandra.index.sasi.disk.OnDiskIndexBuilder;
-import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.schema.*;
import org.apache.cassandra.schema.MigrationManager;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -248,7 +244,7 @@ public class SchemaLoader
MigrationManager.announceNewKeyspace(ksm, false);
if (Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false")))
- useCompression(schema);
+ useCompression(schema, compressionParams(CompressionParams.DEFAULT_CHUNK_LENGTH));
}
public static void createKeyspace(String name, KeyspaceParams params)
@@ -329,11 +325,11 @@ public class SchemaLoader
return builder.build();
}
- private static void useCompression(List<KeyspaceMetadata> schema)
+ private static void useCompression(List<KeyspaceMetadata> schema, CompressionParams compressionParams)
{
for (KeyspaceMetadata ksm : schema)
for (TableMetadata cfm : ksm.tablesAndViews())
- MigrationManager.announceTableUpdate(cfm.unbuild().compression(CompressionParams.snappy()).build(), true);
+ MigrationManager.announceTableUpdate(cfm.unbuild().compression(compressionParams.copy()).build(), true);
}
public static TableMetadata.Builder counterCFMD(String ksName, String cfName)
@@ -721,7 +717,7 @@ public static TableMetadata.Builder clusteringSASICFMD(String ksName, String cfN
public static CompressionParams getCompressionParameters(Integer chunkSize)
{
if (Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false")))
- return chunkSize != null ? CompressionParams.snappy(chunkSize) : CompressionParams.snappy();
+ return chunkSize != null ? compressionParams(chunkSize) : compressionParams(CompressionParams.DEFAULT_CHUNK_LENGTH);
return CompressionParams.noCompression();
}
@@ -751,4 +747,26 @@ public static TableMetadata.Builder clusteringSASICFMD(String ksName, String cfN
{
ServerTestUtils.cleanupSavedCaches();
}
+
+ private static CompressionParams compressionParams(int chunkLength)
+ {
+ String algo = System.getProperty("cassandra.test.compression.algo", "lz4").toLowerCase();
+ switch (algo)
+ {
+ case "deflate":
+ return CompressionParams.deflate(chunkLength);
+ case "lz4":
+ return CompressionParams.lz4(chunkLength);
+ case "snappy":
+ return CompressionParams.snappy(chunkLength);
+ case "noop":
+ return CompressionParams.noop();
+ case "zstd":
+ return CompressionParams.zstd(chunkLength);
+ case "none":
+ return CompressionParams.noCompression();
+ default:
+ throw new IllegalArgumentException("Invalid compression algorithm has been provided in cassandra.test.compression system property: " + algo);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org