You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by dc...@apache.org on 2025/01/24 23:05:32 UTC
(cassandra) branch cassandra-5.0 updated (751d3f24cf0 -> 975c309eee8)
This is an automated email from the ASF dual-hosted git repository.
dcapwell pushed a change to branch cassandra-5.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git
from 751d3f24cf0 Make sure we can set parameters when configuring CassandraCIDRAuthorizer
new 3ddccf45213 CBUtil serialization of UTF8 does not handle all UTF8 properly
new 935effb7e06 Merge branch 'cassandra-4.0' into cassandra-4.1
new 975c309eee8 Merge branch 'cassandra-4.1' into cassandra-5.0
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGES.txt | 1 +
src/java/org/apache/cassandra/db/TypeSizes.java | 10 ++++
.../org/apache/cassandra/gms/VersionedValue.java | 16 +++++++
.../org/apache/cassandra/transport/CBUtil.java | 23 ++++++++--
.../apache/cassandra/gms/VersionedValueTest.java | 53 ++++++++++++++++++++++
.../apache/cassandra/io/IVersionedSerializers.java | 35 +++++++-------
.../org/apache/cassandra/transport/CBUtilTest.java | 19 ++++++++
.../cassandra/utils/AbstractTypeGenerators.java | 22 +++++++++
.../cassandra/utils/CassandraGenerators.java | 40 +++++++++++++++-
9 files changed, 193 insertions(+), 26 deletions(-)
create mode 100644 test/unit/org/apache/cassandra/gms/VersionedValueTest.java
copy src/java/org/apache/cassandra/utils/FailingConsumer.java => test/unit/org/apache/cassandra/io/IVersionedSerializers.java (50%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
(cassandra) 01/01: Merge branch 'cassandra-4.1' into cassandra-5.0
Posted by dc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dcapwell pushed a commit to branch cassandra-5.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 975c309eee89a470f23b48a5b2b44e130129a16c
Merge: 751d3f24cf0 935effb7e06
Author: David Capwell <dc...@apache.org>
AuthorDate: Fri Jan 24 15:00:04 2025 -0800
Merge branch 'cassandra-4.1' into cassandra-5.0
CHANGES.txt | 1 +
src/java/org/apache/cassandra/db/TypeSizes.java | 10 ++++
.../org/apache/cassandra/gms/VersionedValue.java | 16 +++++++
.../org/apache/cassandra/transport/CBUtil.java | 23 ++++++++--
.../apache/cassandra/gms/VersionedValueTest.java | 53 ++++++++++++++++++++++
.../apache/cassandra/io/IVersionedSerializers.java | 39 ++++++++++++++++
.../org/apache/cassandra/transport/CBUtilTest.java | 19 ++++++++
.../cassandra/utils/AbstractTypeGenerators.java | 22 +++++++++
.../cassandra/utils/CassandraGenerators.java | 40 +++++++++++++++-
9 files changed, 216 insertions(+), 7 deletions(-)
diff --cc CHANGES.txt
index 4e36bfd35f9,baa7eee95c8..a187b48c99e
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -14,10 -2,13 +14,11 @@@ Merged from 4.1
* Run audit_logging_options through santiation and validation on startup (CASSANDRA-20208)
* Enforce CQL message size limit on multiframe messages (CASSANDRA-20052)
* Add nodetool checktokenmetadata command that checks TokenMetadata is insync with Gossip endpointState (CASSANDRA-18758)
- * Backport Java 11 support for Simulator (CASSANDRA-17178/CASSANDRA-19935)
* Equality check for Paxos.Electorate should not depend on collection types (CASSANDRA-19935)
- * Fix race condition in DecayingEstimatedHistogramReservoir during rescale (CASSANDRA-19365)
Merged from 4.0:
+ * CBUtil serialization of UTF8 does not handle all UTF8 properly (CASSANDRA-20234)
* Make hint expiry use request start time rather than timeout time for TTL (CASSANDRA-20014)
- * Do not attach rows and partitions to CheckForAbort when already attached (CASSANDRA-20135)
+ * Do not attach rows and partitions to QueryCancellationChecker when already attached (CASSANDRA-20135)
* Allow hint delivery during schema mismatch (CASSANDRA-20188)
* IndexOutOfBoundsException when accessing partition where the column was deleted (CASSANDRA-20108)
* Enhance CQLSSTableWriter to notify clients on sstable production (CASSANDRA-19800)
diff --cc src/java/org/apache/cassandra/gms/VersionedValue.java
index 53b38e1bdb2,ecaab6b81a4..fd763fe1c6e
--- a/src/java/org/apache/cassandra/gms/VersionedValue.java
+++ b/src/java/org/apache/cassandra/gms/VersionedValue.java
@@@ -17,11 -17,9 +17,12 @@@
*/
package org.apache.cassandra.gms;
-import java.io.*;
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
import java.net.InetAddress;
import java.util.Collection;
++import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
@@@ -126,6 -117,6 +127,21 @@@ public class VersionedValue implements
return "Value(" + value + ',' + version + ')';
}
++ @Override
++ public boolean equals(Object o)
++ {
++ if (this == o) return true;
++ if (o == null || getClass() != o.getClass()) return false;
++ VersionedValue that = (VersionedValue) o;
++ return version == that.version && value.equals(that.value);
++ }
++
++ @Override
++ public int hashCode()
++ {
++ return Objects.hash(version, value);
++ }
++
public byte[] toBytes()
{
return value.getBytes(ISO_8859_1);
diff --cc src/java/org/apache/cassandra/transport/CBUtil.java
index d28d9c189e2,dab6b49d9ca..e9bafe5cfad
--- a/src/java/org/apache/cassandra/transport/CBUtil.java
+++ b/src/java/org/apache/cassandra/transport/CBUtil.java
@@@ -38,8 -39,8 +38,7 @@@ import io.netty.buffer.ByteBufUtil
import io.netty.buffer.PooledByteBufAllocator;
import io.netty.buffer.UnpooledByteBufAllocator;
import io.netty.util.concurrent.FastThreadLocal;
-import org.apache.cassandra.config.Config;
import org.apache.cassandra.db.ConsistencyLevel;
- import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.TimeUUID;
diff --cc test/unit/org/apache/cassandra/gms/VersionedValueTest.java
index 00000000000,00000000000..ac47ee5496c
new file mode 100644
--- /dev/null
+++ b/test/unit/org/apache/cassandra/gms/VersionedValueTest.java
@@@ -1,0 -1,0 +1,53 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.cassandra.gms;
++
++import org.junit.Test;
++
++import accord.utils.Gen;
++import org.apache.cassandra.db.TypeSizes;
++import org.apache.cassandra.io.IVersionedSerializers;
++import org.apache.cassandra.io.util.DataOutputBuffer;
++import org.apache.cassandra.net.MessagingService;
++import org.apache.cassandra.utils.CassandraGenerators;
++import org.apache.cassandra.utils.Generators;
++
++import static accord.utils.Property.qt;
++
++public class VersionedValueTest
++{
++ @Test
++ public void serde()
++ {
++ DataOutputBuffer buffer = new DataOutputBuffer();
++ qt().forAll(values()).check(state -> {
++ for (MessagingService.Version version : MessagingService.Version.supportedVersions())
++ IVersionedSerializers.testSerde(buffer, VersionedValue.serializer, state, version.value);
++ });
++ }
++
++ private static Gen<VersionedValue> values()
++ {
++ return Generators.toGen(CassandraGenerators.partitioners().flatMap(CassandraGenerators::gossipStatusValue))
++ // when status == "" this returns null... skip
++ .filter(vv -> vv != null)
++ // sometimes the text is too big, must not be larger than Short.MAX_VALUE
++ .filter(vv -> TypeSizes.encodedUTF8Length(vv.value) <= Short.MAX_VALUE);
++ }
++}
diff --cc test/unit/org/apache/cassandra/io/IVersionedSerializers.java
index 00000000000,00000000000..e17e0b7ce2e
new file mode 100644
--- /dev/null
+++ b/test/unit/org/apache/cassandra/io/IVersionedSerializers.java
@@@ -1,0 -1,0 +1,39 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.cassandra.io;
++
++import java.io.IOException;
++
++import org.apache.cassandra.io.util.DataInputBuffer;
++import org.apache.cassandra.io.util.DataOutputBuffer;
++import org.assertj.core.api.Assertions;
++
++public class IVersionedSerializers
++{
++ public static <T> void testSerde(DataOutputBuffer output, IVersionedSerializer<T> serializer, T input, int version) throws IOException
++ {
++ output.clear();
++ long expectedSize = serializer.serializedSize(input, version);
++ serializer.serialize(input, output, version);
++ Assertions.assertThat(output.getLength()).describedAs("The serialized size and bytes written do not match").isEqualTo(expectedSize);
++ DataInputBuffer in = new DataInputBuffer(output.unsafeGetBufferAndFlip(), false);
++ T read = serializer.deserialize(in, version);
++ Assertions.assertThat(read).describedAs("The deserialized output does not match the serialized input").isEqualTo(input);
++ }
++}
diff --cc test/unit/org/apache/cassandra/transport/CBUtilTest.java
index c20efec8f83,9e0cd74b838..c74a8a50a81
--- a/test/unit/org/apache/cassandra/transport/CBUtilTest.java
+++ b/test/unit/org/apache/cassandra/transport/CBUtilTest.java
@@@ -22,9 -22,13 +22,13 @@@ import org.junit.After
import org.junit.Assert;
import org.junit.Test;
++import accord.utils.Gens;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.PooledByteBufAllocator;
+ import org.assertj.core.api.Assertions;
-import org.quicktheories.generators.SourceDSL;
+
-import static org.quicktheories.QuickTheory.qt;
++import static accord.utils.Property.qt;
public class CBUtilTest
{
@@@ -38,6 -42,21 +42,21 @@@
buf.release(buf.refCnt());
}
+ @Test
+ public void stringList()
+ {
- qt().forAll(SourceDSL.lists().of(SourceDSL.strings().allPossible().ofLengthBetween(0, 20)).ofSizeBetween(0, 10)).checkAssert(list -> {
++ qt().forAll(Gens.lists(Gens.strings().all().ofLengthBetween(0, 20)).ofSizeBetween(0, 10)).check(list -> {
+ int expectedSize = CBUtil.sizeOfStringList(list);
+ ByteBuf body = CBUtil.allocator.buffer(expectedSize * 2);
+ CBUtil.writeStringList(list, body);
+ Assertions.assertThat(body.readableBytes()).isEqualTo(expectedSize);
+ // In CASSANDRA-20234 the sizeOf method now reflects the write method, but the work to do this in read
+ // was not done; read took a stance that it wants to limit things in CASSANDRA-8101 but that never made
+ // it to the other methods... write and read are not compatable with the full domain of UTF-8, and fixing
+ // that was higher bar than CASSANDRA-20234 wanted to tackle out of fear of unknown regressions it would cause.
+ });
+ }
+
@Test
public void writeAndReadString()
{
diff --cc test/unit/org/apache/cassandra/utils/AbstractTypeGenerators.java
index 62537208ec7,12189ed6972..8bfac9701d3
--- a/test/unit/org/apache/cassandra/utils/AbstractTypeGenerators.java
+++ b/test/unit/org/apache/cassandra/utils/AbstractTypeGenerators.java
@@@ -99,10 -94,9 +99,11 @@@ import org.quicktheories.core.Randomnes
import org.quicktheories.generators.SourceDSL;
import org.quicktheories.impl.JavaRandom;
++import static org.apache.cassandra.utils.AbstractTypeGenerators.TypeKind.COUNTER;
import static org.apache.cassandra.utils.Generators.IDENTIFIER_GEN;
+import static org.apache.cassandra.utils.Generators.filter;
-@SuppressWarnings({ "unchecked", "rawtypes" })
+@SuppressWarnings({"unchecked", "rawtypes"})
public final class AbstractTypeGenerators
{
private final static Logger logger = LoggerFactory.getLogger(AbstractTypeGenerators.class);
@@@ -228,295 -199,13 +229,316 @@@
return PRIMITIVE_TYPE_GEN;
}
- public static Gen<AbstractType<?>> primitiveTypeGen(AbstractType<?>... without)
+ public static Set<Class<? extends AbstractType<?>>> UNSAFE_EQUALITY = ImmutableSet.of(EmptyType.class,
+ DurationType.class,
+ DecimalType.class,
+ CounterColumnType.class);
+
+ public static <T> Releaser overridePrimitiveTypeSupport(AbstractType<T> type, TypeSupport<T> support)
+ {
+ if (!PRIMITIVE_TYPE_DATA_GENS.containsKey(type))
+ throw new IllegalArgumentException("Type " + type.asCQL3Type() + " is not a primitive");
+ TypeSupport<?> original = PRIMITIVE_TYPE_DATA_GENS.get(type);
+ PRIMITIVE_TYPE_DATA_GENS.put(type, support);
+ return () -> PRIMITIVE_TYPE_DATA_GENS.put(type, original);
+ }
+
+ public static TypeGenBuilder withoutUnsafeEquality()
+ {
+ // make sure to keep UNSAFE_EQUALITY in-sync
+ return AbstractTypeGenerators.builder()
+ .withoutEmpty()
+ .withoutPrimitive(DurationType.instance)
+ // decimal "normalizes" the data to compare, so primary columns "may" mutate the data, causing missmatches
+ // see CASSANDRA-18530
+ .withoutPrimitive(DecimalType.instance)
+ // counters are only for top level
+ .withoutTypeKinds(TypeKind.COUNTER);
+ }
+
+ public interface Releaser extends AutoCloseable
+ {
+ @Override
+ void close();
+ }
+
+ public static class TypeGenBuilder
+ {
+ private int maxDepth = 3;
+ private EnumSet<TypeKind> kinds;
+ private Gen<TypeKind> typeKindGen;
+ private Gen<Integer> defaultSizeGen = VERY_SMALL_POSITIVE_SIZE_GEN;
+ private Gen<Integer> vectorSizeGen, tupleSizeGen, udtSizeGen, compositeSizeGen;
+ private Gen<AbstractType<?>> primitiveGen = PRIMITIVE_TYPE_GEN, compositeElementGen;
+ private Gen<String> userTypeKeyspaceGen = IDENTIFIER_GEN;
+ private Function<Integer, Gen<AbstractType<?>>> defaultSetKeyFunc;
+ private Predicate<AbstractType<?>> typeFilter = null;
+ private Gen<String> udtName = null;
+
+ public TypeGenBuilder()
+ {
+ }
+
+ public TypeGenBuilder(TypeGenBuilder other)
+ {
+ maxDepth = other.maxDepth;
+ kinds = other.kinds == null ? null : EnumSet.copyOf(other.kinds);
+ typeKindGen = other.typeKindGen;
+ defaultSizeGen = other.defaultSizeGen;
+ vectorSizeGen = other.vectorSizeGen;
+ tupleSizeGen = other.tupleSizeGen;
+ udtName = other.udtName;
+ udtSizeGen = other.udtSizeGen;
+ primitiveGen = other.primitiveGen;
+ userTypeKeyspaceGen = other.userTypeKeyspaceGen;
+ defaultSetKeyFunc = other.defaultSetKeyFunc;
+ compositeElementGen = other.compositeElementGen;
+ compositeSizeGen = other.compositeSizeGen;
+ typeFilter = other.typeFilter;
+ }
+
+ public TypeGenBuilder withTypeFilter(Predicate<AbstractType<?>> fn)
+ {
+ typeFilter = fn;
+ return this;
+ }
+
+ public TypeGenBuilder withCompositeElementGen(Gen<AbstractType<?>> gen)
+ {
+ this.compositeElementGen = gen;
+ return this;
+ }
+
+ @SuppressWarnings("unused")
+ public TypeGenBuilder withDefaultSetKey(Function<Integer, Gen<AbstractType<?>>> mapKeyFunc)
+ {
+ this.defaultSetKeyFunc = mapKeyFunc;
+ return this;
+ }
+
+ public TypeGenBuilder withDefaultSetKey(TypeGenBuilder builder)
+ {
+ this.defaultSetKeyFunc = builder::buildRecursive;
+ return this;
+ }
+
+ public TypeGenBuilder withUserTypeKeyspace(String keyspace)
+ {
+ userTypeKeyspaceGen = SourceDSL.arbitrary().constant(keyspace);
+ return this;
+ }
+
+ public TypeGenBuilder withDefaultSizeGen(int size)
+ {
+ return withDefaultSizeGen(SourceDSL.arbitrary().constant(size));
+ }
+
+ public TypeGenBuilder withDefaultSizeGen(Gen<Integer> sizeGen)
+ {
+ this.defaultSizeGen = sizeGen;
+ return this;
+ }
+
+ @SuppressWarnings("unused")
+ public TypeGenBuilder withVectorSizeGen(Gen<Integer> sizeGen)
+ {
+ this.vectorSizeGen = sizeGen;
+ return this;
+ }
+
+ @SuppressWarnings("unused")
+ public TypeGenBuilder withTupleSizeGen(Gen<Integer> sizeGen)
+ {
+ this.tupleSizeGen = sizeGen;
+ return this;
+ }
+
+ @SuppressWarnings("unused")
+ public TypeGenBuilder withUDTSizeGen(Gen<Integer> sizeGen)
+ {
+ this.udtSizeGen = sizeGen;
+ return this;
+ }
+
+ @SuppressWarnings("unused")
+ public TypeGenBuilder withCompositeSizeGen(Gen<Integer> sizeGen)
+ {
+ this.compositeSizeGen = sizeGen;
+ return this;
+ }
+
+ public TypeGenBuilder withoutEmpty()
+ {
+ return withoutPrimitive(EmptyType.instance);
+ }
+
+ public TypeGenBuilder withoutPrimitive(AbstractType<?> instance)
+ {
+ if (!PRIMITIVE_TYPE_DATA_GENS.containsKey(instance))
+ throw new IllegalArgumentException("Type " + instance + " is not a primitive type, or PRIMITIVE_TYPE_DATA_GENS needs to add support");
+ primitiveGen = filter(primitiveGen, t -> t != instance);
+ return this;
+ }
+
+ @SuppressWarnings("unused")
+ public TypeGenBuilder withPrimitives(AbstractType<?> first, AbstractType<?>... remaining)
+ {
+ // any previous filters will be ignored...
+ primitiveGen = SourceDSL.arbitrary().pick(ArrayUtils.add(remaining, first));
+ return this;
+ }
+
+ public TypeGenBuilder withMaxDepth(int value)
+ {
+ this.maxDepth = value;
+ return this;
+ }
+
+ public TypeGenBuilder withoutTypeKinds(TypeKind... values)
+ {
+ checkTypeKindValues();
+ for (TypeKind kind : values)
+ kinds.remove(kind);
+ return this;
+ }
+
+ public TypeGenBuilder withTypeKinds(TypeKind... values)
+ {
+ checkTypeKindValues();
+ kinds.clear();
+ Collections.addAll(kinds, values);
+ return this;
+ }
+
+ private void checkTypeKindValues()
+ {
+ if (typeKindGen != null)
+ throw new IllegalArgumentException("Mixed both generator and individaul values for type kind");
+ if (kinds == null)
+ kinds = EnumSet.allOf(TypeKind.class);
+ }
+
+ public TypeGenBuilder withTypeKinds(Gen<TypeKind> typeKindGen)
+ {
+ if (kinds != null)
+ throw new IllegalArgumentException("Mixed both generator and individaul values for type kind");
+ this.typeKindGen = Objects.requireNonNull(typeKindGen);
+ return this;
+ }
+
+ public TypeGenBuilder withUDTNames(Gen<String> udtName)
+ {
+ this.udtName = udtName;
+ return this;
+ }
+
+ public Gen<AbstractType<?>> build()
+ {
+ if (udtName == null)
+ udtName = Generators.unique(IDENTIFIER_GEN);
+ // strip out the package to make it easier to read
+ // type parser assumes this package when one isn't provided, so this does not corrupt the type conversion
+ return buildRecursive(maxDepth).describedAs(t -> t.asCQL3Type().toString().replaceAll("org.apache.cassandra.db.marshal.", ""));
+ }
+
+ private Gen<AbstractType<?>> buildRecursive(int maxDepth)
+ {
+ if (udtName == null)
+ udtName = Generators.unique(IDENTIFIER_GEN);
+ Gen<TypeKind> kindGen;
+ if (typeKindGen != null)
+ kindGen = typeKindGen;
+ else if (kinds != null)
+ {
+ ArrayList<TypeKind> ts = new ArrayList<>(kinds);
+ Collections.sort(ts);
+ kindGen = SourceDSL.arbitrary().pick(ts);
+ }
+ else
+ kindGen = SourceDSL.arbitrary().enumValues(TypeKind.class);
+ return buildRecursive(maxDepth, maxDepth, kindGen, BOOLEAN_GEN);
+ }
+
+ private Gen<AbstractType<?>> buildRecursive(int maxDepth, int level, Gen<TypeKind> typeKindGen, Gen<Boolean> multiCellGen)
+ {
+ if (level == -1)
+ return primitiveGen;
+ assert level >= 0 : "max depth must be positive or zero; given " + level;
+ boolean atBottom = level == 0;
+ boolean atTop = maxDepth == level;
+ Gen<AbstractType<?>> gen = rnd -> {
+ Supplier<Gen<AbstractType<?>>> next = () -> atBottom ? primitiveGen : buildRecursive(maxDepth, level - 1, typeKindGen, multiCellGen);
+
+ // figure out type to get
+ TypeKind kind = typeKindGen.generate(rnd);
+ // counters are only allowed at the top level
+ while (!atTop && kind == TypeKind.COUNTER)
+ kind = typeKindGen.generate(rnd);
+ switch (kind)
+ {
+ case PRIMITIVE:
+ return primitiveGen.generate(rnd);
+ case SET:
+ if (defaultSetKeyFunc != null)
+ return setTypeGen(defaultSetKeyFunc.apply(level - 1), multiCellGen).generate(rnd);
+ return setTypeGen(next.get(), multiCellGen).generate(rnd);
+ case LIST:
+ return listTypeGen(next.get(), multiCellGen).generate(rnd);
+ case MAP:
+ if (defaultSetKeyFunc != null)
+ return mapTypeGen(defaultSetKeyFunc.apply(level - 1), next.get(), multiCellGen).generate(rnd);
+ return mapTypeGen(next.get(), next.get(), multiCellGen).generate(rnd);
+ case TUPLE:
+ return tupleTypeGen(atBottom ? primitiveGen : buildRecursive(maxDepth, level - 1, typeKindGen, SourceDSL.arbitrary().constant(false)), tupleSizeGen != null ? tupleSizeGen : defaultSizeGen).generate(rnd);
+ case UDT:
+ return userTypeGen(next.get(), udtSizeGen != null ? udtSizeGen : defaultSizeGen, userTypeKeyspaceGen, udtName, multiCellGen).generate(rnd);
+ case VECTOR:
+ {
+ Gen<Integer> sizeGen = vectorSizeGen != null ? vectorSizeGen : defaultSizeGen;
+ return vectorTypeGen(next.get().map(AbstractType::freeze), sizeGen).generate(rnd);
+ }
+ case COMPOSITE:
+ return compositeTypeGen(compositeElementGen != null ? compositeElementGen : next.get(), compositeSizeGen != null ? compositeSizeGen : defaultSizeGen).generate(rnd);
+ case DYNAMIC_COMPOSITE:
+ Gen<Byte> aliasGen = Generators.letterOrDigit().map(c -> (byte) c.charValue());
+ // stores alias names by class and not what is actually valid by cql... so only primitive types match!
+ return dynamicCompositeGen(primitiveGen, aliasGen, defaultSizeGen).generate(rnd);
+ case COUNTER:
+ return CounterColumnType.instance;
+ default:
+ throw new IllegalArgumentException("Unknown kind: " + kind);
+ }
+ };
+ return typeFilter == null ? gen : filter(gen, typeFilter);
+ }
+ }
+
+ public static TypeGenBuilder builder()
{
- List<AbstractType<?>> types = new ArrayList<>(PRIMITIVE_TYPE_DATA_GENS.keySet());
- for (AbstractType<?> t : without)
- types.remove(t);
- return SourceDSL.arbitrary().pick(types);
+ return new TypeGenBuilder();
}
+
++ /**
++ * Similar to {@link #typeGen()} but removes types that are known to be problematic in some cases and limits the depth
++ * of the type tree to avoid cell constratins.
++ */
++ public static Gen<AbstractType<?>> safeTypeGen()
++ {
++ TypeGenBuilder baseline = AbstractTypeGenerators.builder()
++ // neither of these types support the property
++ // expected == fromComparableBytes(asComparableBytes(expected))
++ // so rather than having tests try to skip... just avoid those types!
++ .withoutEmpty()
++ .withoutTypeKinds(COUNTER)
++ .withoutPrimitive(DecimalType.instance)
++ // its ordering is special...
++ .withoutPrimitive(DurationType.instance);
++ // composite requires all elements fit into Short.MAX_VALUE bytes
++ // so try to limit the possible expansion of types
++ return baseline.withCompositeElementGen(new TypeGenBuilder(baseline).withDefaultSizeGen(1).withMaxDepth(1).build())
++ .build();
++ }
++
public static Gen<AbstractType<?>> typeGen()
{
return typeGen(3);
diff --cc test/unit/org/apache/cassandra/utils/CassandraGenerators.java
index c6ab6b9ac45,b5cf3156ec6..b064ec98fc7
--- a/test/unit/org/apache/cassandra/utils/CassandraGenerators.java
+++ b/test/unit/org/apache/cassandra/utils/CassandraGenerators.java
@@@ -545,197 -318,6 +545,233 @@@ public final class CassandraGenerator
}, true);
}
+ public static Gen<Token> murmurToken()
+ {
+ Constraint token = Constraint.between(Long.MIN_VALUE, Long.MAX_VALUE);
+ return rs -> new Murmur3Partitioner.LongToken(rs.next(token));
+ }
+
+ public static Gen<Token> murmurTokenIn(Range<Token> range)
+ {
+ // left exclusive, right inclusive
+ if (range.isWrapAround())
+ {
+ List<Range<Token>> unwrap = range.unwrap();
+ return rs -> {
+ Range<Token> subRange = unwrap.get(Math.toIntExact(rs.next(Constraint.between(0, unwrap.size() - 1))));
+ long end = ((Murmur3Partitioner.LongToken) subRange.right).token;
+ if (end == Long.MIN_VALUE)
+ end = Long.MAX_VALUE;
+ Constraint token = Constraint.between(((Murmur3Partitioner.LongToken) subRange.left).token + 1, end);
+ return new Murmur3Partitioner.LongToken(rs.next(token));
+ };
+ }
+ else
+ {
+ Constraint token = Constraint.between(((Murmur3Partitioner.LongToken) range.left).token + 1, ((Murmur3Partitioner.LongToken) range.right).token);
+ return rs -> new Murmur3Partitioner.LongToken(rs.next(token));
+ }
+ }
+
+ public static Gen<Token> byteOrderToken()
+ {
+ Constraint size = Constraint.between(0, 10);
+ Constraint byteRange = Constraint.between(Byte.MIN_VALUE, Byte.MAX_VALUE);
+ return rs -> {
+ byte[] token = new byte[Math.toIntExact(rs.next(size))];
+ for (int i = 0; i < token.length; i++)
+ token[i] = (byte) rs.next(byteRange);
+ return new ByteOrderedPartitioner.BytesToken(token);
+ };
+ }
+
+ public static Gen<Token> randomPartitionerToken()
+ {
+ Constraint domain = Constraint.none();
+ return rs -> new RandomPartitioner.BigIntegerToken(BigInteger.valueOf(rs.next(domain)));
+ }
+
+ public static Gen<Token> localPartitionerToken(LocalPartitioner partitioner)
+ {
+ Gen<ByteBuffer> bytes = AbstractTypeGenerators.getTypeSupport(partitioner.getTokenValidator()).bytesGen();
+ return rs -> partitioner.getToken(bytes.generate(rs));
+ }
+
+ public static Gen<Token> orderPreservingToken()
+ {
+ Gen<String> string = Generators.utf8(0, 10);
+ return rs -> new OrderPreservingPartitioner.StringToken(string.generate(rs));
+ }
+
+ public static Gen<Token> tokensInRange(Range<Token> range)
+ {
+ IPartitioner partitioner = range.left.getPartitioner();
+ if (partitioner instanceof Murmur3Partitioner) return murmurTokenIn(range);
+ throw new UnsupportedOperationException("Unsupported partitioner: " + partitioner.getClass());
+ }
+
+ public static Gen<Token> token(IPartitioner partitioner)
+ {
+ if (partitioner instanceof Murmur3Partitioner) return murmurToken();
+ if (partitioner instanceof ByteOrderedPartitioner) return byteOrderToken();
+ if (partitioner instanceof RandomPartitioner) return randomPartitionerToken();
+ if (partitioner instanceof LocalPartitioner) return localPartitionerToken((LocalPartitioner) partitioner);
+ if (partitioner instanceof OrderPreservingPartitioner) return orderPreservingToken();
+ throw new UnsupportedOperationException("Unsupported partitioner: " + partitioner.getClass());
+ }
+
+ public static Gen<? extends Collection<Token>> tokens(IPartitioner partitioner)
+ {
+ Gen<Token> tokenGen = token(partitioner);
+ return SourceDSL.lists().of(tokenGen).ofSizeBetween(1, 16);
+ }
+
++ public static Gen<IPartitioner> localPartitioner()
++ {
++ return AbstractTypeGenerators.safeTypeGen().map(LocalPartitioner::new);
++ }
++
++ private enum SupportedPartitioners
++ {
++ Murmur(ignore -> Murmur3Partitioner.instance),
++ ByteOrdered(ignore -> ByteOrderedPartitioner.instance),
++ Random(ignore -> RandomPartitioner.instance),
++ Local(localPartitioner()),
++ OrderPreserving(ignore -> OrderPreservingPartitioner.instance);
++
++ private final Gen<IPartitioner> partitioner;
++
++ SupportedPartitioners(Gen<IPartitioner> partitionerGen)
++ {
++ partitioner = partitionerGen;
++ }
++
++ public Gen<IPartitioner> partitioner()
++ {
++ return partitioner;
++ }
++ }
++
++ public static Gen<IPartitioner> partitioners()
++ {
++ return SourceDSL.arbitrary().enumValues(SupportedPartitioners.class)
++ .flatMap(SupportedPartitioners::partitioner);
++ }
++
+ public static Gen<HeartBeatState> heartBeatStates()
+ {
+ Constraint generationDomain = Constraint.between(0, Integer.MAX_VALUE);
+ Constraint versionDomain = Constraint.between(-1, Integer.MAX_VALUE);
+ return rs -> new HeartBeatState(Math.toIntExact(rs.next(generationDomain)), Math.toIntExact(rs.next(versionDomain)));
+ }
+
+ private static Gen<Map<ApplicationState, VersionedValue>> gossipApplicationStates()
+ {
+ //TODO support all application states...
+ // atm only used by a single test, which only looks at status
+ Gen<Boolean> statusWithPort = SourceDSL.booleans().all();
+ Gen<VersionedValue> statusGen = gossipStatusValue();
+
+ return rs -> {
+ ApplicationState statusState = statusWithPort.generate(rs) ? ApplicationState.STATUS_WITH_PORT : ApplicationState.STATUS;
+ VersionedValue vv = statusGen.generate(rs);
+ if (vv == null) return ImmutableMap.of();
+ return ImmutableMap.of(statusState, vv);
+ };
+ }
+
+ private static Gen<String> gossipStatus()
+ {
+ return SourceDSL.arbitrary()
+ .pick(VersionedValue.STATUS_NORMAL,
+ VersionedValue.STATUS_BOOTSTRAPPING_REPLACE,
+ VersionedValue.STATUS_BOOTSTRAPPING,
+ VersionedValue.STATUS_MOVING,
+ VersionedValue.STATUS_LEAVING,
+ VersionedValue.STATUS_LEFT,
+
+ //TODO would be good to prefix with STATUS_ like others
+ VersionedValue.REMOVING_TOKEN,
+ VersionedValue.REMOVED_TOKEN,
+ VersionedValue.HIBERNATE + VersionedValue.DELIMITER + true,
+ VersionedValue.HIBERNATE + VersionedValue.DELIMITER + false,
+ VersionedValue.SHUTDOWN + VersionedValue.DELIMITER + true,
+ VersionedValue.SHUTDOWN + VersionedValue.DELIMITER + false,
+ ""
+ );
+ }
+
- private static Gen<VersionedValue> gossipStatusValue()
++ public static Gen<VersionedValue> gossipStatusValue()
++ {
++ return gossipStatusValue(DatabaseDescriptor.getPartitioner());
++ }
++
++ public static Gen<VersionedValue> gossipStatusValue(IPartitioner partitioner)
+ {
- IPartitioner partitioner = DatabaseDescriptor.getPartitioner();
+ Gen<String> statusGen = gossipStatus();
+ Gen<Token> tokenGen = token(partitioner);
+ Gen<? extends Collection<Token>> tokensGen = tokens(partitioner);
+ Gen<InetAddress> addressGen = Generators.INET_ADDRESS_GEN;
+ Gen<InetAddressAndPort> addressAndGenGen = INET_ADDRESS_AND_PORT_GEN;
+ Gen<Boolean> bool = SourceDSL.booleans().all();
+ Constraint millis = Constraint.between(0, Long.MAX_VALUE);
+ Constraint version = Constraint.between(0, Integer.MAX_VALUE);
+ Gen<UUID> hostId = Generators.UUID_RANDOM_GEN;
+ VersionedValue.VersionedValueFactory factory = new VersionedValue.VersionedValueFactory(partitioner);
+ return rs -> {
+ String status = statusGen.generate(rs);
+ switch (status)
+ {
+ case "":
+ return null;
+ case VersionedValue.STATUS_NORMAL:
+ return factory.normal(tokensGen.generate(rs)).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.STATUS_BOOTSTRAPPING:
+ return factory.bootstrapping(tokensGen.generate(rs)).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.STATUS_BOOTSTRAPPING_REPLACE:
+ if (bool.generate(rs)) return factory.bootReplacingWithPort(addressAndGenGen.generate(rs)).withVersion(Math.toIntExact(rs.next(version)));
+ else return factory.bootReplacing(addressGen.generate(rs)).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.STATUS_MOVING:
+ return factory.moving(tokenGen.generate(rs)).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.STATUS_LEAVING:
+ return factory.leaving(tokensGen.generate(rs)).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.STATUS_LEFT:
+ return factory.left(tokensGen.generate(rs), rs.next(millis)).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.REMOVING_TOKEN:
+ return factory.removingNonlocal(hostId.generate(rs)).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.REMOVED_TOKEN:
+ return factory.removedNonlocal(hostId.generate(rs), rs.next(millis)).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.HIBERNATE + VersionedValue.DELIMITER + true:
+ return factory.hibernate(true).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.HIBERNATE + VersionedValue.DELIMITER + false:
+ return factory.hibernate(false).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.SHUTDOWN + VersionedValue.DELIMITER + true:
+ return factory.shutdown(true).withVersion(Math.toIntExact(rs.next(version)));
+ case VersionedValue.SHUTDOWN + VersionedValue.DELIMITER + false:
+ return factory.shutdown(false).withVersion(Math.toIntExact(rs.next(version)));
+ default:
+ throw new AssertionError("Unexpected status: " + status);
+ }
+ };
+ }
+
+ public static Gen<EndpointState> endpointStates()
+ {
+ Gen<HeartBeatState> hbGen = heartBeatStates();
+ Gen<Map<ApplicationState, VersionedValue>> appStates = gossipApplicationStates();
+ Gen<Boolean> alive = SourceDSL.booleans().all();
+ Constraint updateTimestamp = Constraint.between(0, Long.MAX_VALUE);
+ return rs -> {
+ EndpointState state = new EndpointState(hbGen.generate(rs));
+ Map<ApplicationState, VersionedValue> map = appStates.generate(rs);
+ if (!map.isEmpty()) state.addApplicationStates(map);
+ if (alive.generate(rs)) state.markAlive();
+ else state.markDead();
+ state.unsafeSetUpdateTimestamp(rs.next(updateTimestamp));
+ return state;
+ };
+ }
+
public static Gen<Duration> duration()
{
Constraint ints = Constraint.between(0, Integer.MAX_VALUE);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org