You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by dc...@apache.org on 2024/09/06 22:21:32 UTC
(cassandra) branch cassandra-5.0 updated: Add guardrail for enabling usage of VectorType
This is an automated email from the ASF dual-hosted git repository.
dcapwell pushed a commit to branch cassandra-5.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cassandra-5.0 by this push:
new b4894226c5 Add guardrail for enabling usage of VectorType
b4894226c5 is described below
commit b4894226c57bf2cbc29cad1db47938c9e5ef0a7c
Author: Abe Ratnofsky <ab...@aber.io>
AuthorDate: Fri Sep 6 12:43:24 2024 -0700
Add guardrail for enabling usage of VectorType
patch by Abe Ratnofsky; reviewed by Caleb Rackliffe, David Capwell for CASSANDRA-19903
---
CHANGES.txt | 1 +
src/java/org/apache/cassandra/config/Config.java | 1 +
.../apache/cassandra/config/GuardrailsOptions.java | 15 +++
src/java/org/apache/cassandra/cql3/CQL3Type.java | 2 +
.../apache/cassandra/db/guardrails/Guardrails.java | 25 ++++
.../cassandra/db/guardrails/GuardrailsConfig.java | 12 ++
.../cassandra/db/guardrails/GuardrailsMBean.java | 10 ++
.../guardrails/GuardrailVectorDimensionsTest.java | 105 +++++++++------
.../guardrails/GuardrailVectorTypeEnabledTest.java | 147 +++++++++++++++++++++
9 files changed, 276 insertions(+), 42 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 7e5e00b558..d9242e0c0a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
5.0.1
+ * Add guardrail for enabling usage of VectorType (CASSANDRA-19903)
* Set executable flag for shell scripts in .build directory for source artifact (CASSANDRA-19896)
* Throw ConfigurationException when commitlog_disk_access_mode is direct but direct IO is not supported by filesystem (CASSANDRA-19812)
* Minor improvements in Cassandra shutdown and startup logs (CASSANDRA-19818)
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index 8fbb1579aa..63ce5355c5 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -892,6 +892,7 @@ public class Config
public volatile int items_per_collection_fail_threshold = -1;
public volatile int fields_per_udt_warn_threshold = -1;
public volatile int fields_per_udt_fail_threshold = -1;
+ public volatile boolean vector_type_enabled = true;
public volatile int vector_dimensions_warn_threshold = -1;
public volatile int vector_dimensions_fail_threshold = -1;
public volatile int data_disk_usage_percentage_warn_threshold = -1;
diff --git a/src/java/org/apache/cassandra/config/GuardrailsOptions.java b/src/java/org/apache/cassandra/config/GuardrailsOptions.java
index 504384e0d0..6e82e7f908 100644
--- a/src/java/org/apache/cassandra/config/GuardrailsOptions.java
+++ b/src/java/org/apache/cassandra/config/GuardrailsOptions.java
@@ -1059,6 +1059,21 @@ public class GuardrailsOptions implements GuardrailsConfig
x -> config.non_partition_restricted_index_query_enabled = x);
}
+ @Override
+ public boolean getVectorTypeEnabled()
+ {
+ return config.vector_type_enabled;
+ }
+
+ @Override
+ public void setVectorTypeEnabled(boolean enabled)
+ {
+ updatePropertyWithLogging("vector_type_enabled",
+ enabled,
+ () -> config.vector_type_enabled,
+ x -> config.vector_type_enabled = x);
+ }
+
private static <T> void updatePropertyWithLogging(String propertyName, T newValue, Supplier<T> getter, Consumer<T> setter)
{
T oldValue = getter.get();
diff --git a/src/java/org/apache/cassandra/cql3/CQL3Type.java b/src/java/org/apache/cassandra/cql3/CQL3Type.java
index 304363ad1d..43cc9b3f24 100644
--- a/src/java/org/apache/cassandra/cql3/CQL3Type.java
+++ b/src/java/org/apache/cassandra/cql3/CQL3Type.java
@@ -722,6 +722,7 @@ public interface CQL3Type
{
if (type.isVector())
{
+ Guardrails.vectorTypeEnabled.ensureEnabled(name, state);
int dimensions = ((Vector) type).getType().dimension;
Guardrails.vectorDimensions.guard(dimensions, name, false, state);
}
@@ -920,6 +921,7 @@ public interface CQL3Type
@Override
public void validate(ClientState state, String name)
{
+ Guardrails.vectorTypeEnabled.ensureEnabled(name, state);
Guardrails.vectorDimensions.guard(dimension, name, false, state);
}
diff --git a/src/java/org/apache/cassandra/db/guardrails/Guardrails.java b/src/java/org/apache/cassandra/db/guardrails/Guardrails.java
index f61ae204d1..829dab0562 100644
--- a/src/java/org/apache/cassandra/db/guardrails/Guardrails.java
+++ b/src/java/org/apache/cassandra/db/guardrails/Guardrails.java
@@ -400,6 +400,19 @@ public final class Guardrails implements GuardrailsMBean
: format("User types cannot have more than %s columns, but %s provided for user type %s.",
threshold, value, what));
+ /**
+ * Guardrail on the usage of vector type.
+ *
+ * This may be useful when running clusters that have diverse clients, where some do not support the vector type.
+ * Clients that do not support vector may fail to establish sessions or fail to read results containing vector
+ * columns.
+ */
+ public static final EnableFlag vectorTypeEnabled =
+ new EnableFlag("vector_type_enabled",
+ null,
+ state -> CONFIG_PROVIDER.getOrCreate(state).getVectorTypeEnabled(),
+ "usage of the vector type");
+
/**
* Guardrail on the number of dimensions of vector columns.
*/
@@ -1139,6 +1152,18 @@ public final class Guardrails implements GuardrailsMBean
DEFAULT_CONFIG.setVectorDimensionsThreshold(warn, fail);
}
+ @Override
+ public void setVectorTypeEnabled(boolean enabled)
+ {
+ DEFAULT_CONFIG.setVectorTypeEnabled(enabled);
+ }
+
+ @Override
+ public boolean getVectorTypeEnabled()
+ {
+ return DEFAULT_CONFIG.getVectorTypeEnabled();
+ }
+
@Override
public int getMaximumReplicationFactorWarnThreshold()
{
diff --git a/src/java/org/apache/cassandra/db/guardrails/GuardrailsConfig.java b/src/java/org/apache/cassandra/db/guardrails/GuardrailsConfig.java
index 508ae53fc0..ece387461d 100644
--- a/src/java/org/apache/cassandra/db/guardrails/GuardrailsConfig.java
+++ b/src/java/org/apache/cassandra/db/guardrails/GuardrailsConfig.java
@@ -304,6 +304,18 @@ public interface GuardrailsConfig
*/
int getFieldsPerUDTFailThreshold();
+ /**
+ * @return Whether new columns can be created with vector type
+ */
+ boolean getVectorTypeEnabled();
+
+ /**
+ * Sets whether new columns can be created with vector type
+ *
+ * @param enabled
+ */
+ void setVectorTypeEnabled(boolean enabled);
+
/**
* @return The threshold to warn when creating a vector with more dimensions than threshold.
*/
diff --git a/src/java/org/apache/cassandra/db/guardrails/GuardrailsMBean.java b/src/java/org/apache/cassandra/db/guardrails/GuardrailsMBean.java
index 9a3750bba8..d40092314a 100644
--- a/src/java/org/apache/cassandra/db/guardrails/GuardrailsMBean.java
+++ b/src/java/org/apache/cassandra/db/guardrails/GuardrailsMBean.java
@@ -613,6 +613,16 @@ public interface GuardrailsMBean
*/
void setVectorDimensionsThreshold(int warn, int fail);
+ /**
+ * @param enabled {@code true} if vector type usage is enabled.
+ */
+ void setVectorTypeEnabled(boolean enabled);
+
+ /**
+ * @return {@code true} if vector type usage is enabled.
+ */
+ boolean getVectorTypeEnabled();
+
/**
* @return The threshold to warn when local data disk usage percentage exceeds that threshold.
* Allowed values are in the range {@code [1, 100]}, and -1 means disabled.
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailVectorDimensionsTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailVectorDimensionsTest.java
index 45727272b2..8cd5658768 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailVectorDimensionsTest.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailVectorDimensionsTest.java
@@ -18,6 +18,7 @@
package org.apache.cassandra.db.guardrails;
+import java.util.List;
import java.util.function.Supplier;
import org.junit.Test;
@@ -42,32 +43,37 @@ public class GuardrailVectorDimensionsTest extends ThresholdTester
Guardrails::getVectorDimensionsFailThreshold);
}
- @Test
- public void testCreateTable() throws Throwable
- {
+ static final List<String> CREATE_TABLE_CQL = List.of(
// different poisitions (partition key, clustering key, static column, regular column)
- testCreateTable("CREATE TABLE %s (v vector<int, %%d> PRIMARY KEY)");
- testCreateTable("CREATE TABLE %s (v vector<int, %%d>, c int, PRIMARY KEY(v, c))");
- testCreateTable("CREATE TABLE %s (v vector<int, %%d>, c int, PRIMARY KEY((v, c)))");
- testCreateTable("CREATE TABLE %s (k int, v vector<int, %%d>, PRIMARY KEY(k, v))");
- testCreateTable("CREATE TABLE %s (k int, c int, v vector<int, %%d>, PRIMARY KEY(k, c, v))");
- testCreateTable("CREATE TABLE %s (k int, c int, v vector<int, %%d> static, PRIMARY KEY(k, c))");
- testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v vector<int, %%d>)");
+ "CREATE TABLE %s (v vector<int, %%d> PRIMARY KEY)",
+ "CREATE TABLE %s (v vector<int, %%d>, c int, PRIMARY KEY(v, c))",
+ "CREATE TABLE %s (v vector<int, %%d>, c int, PRIMARY KEY((v, c)))",
+ "CREATE TABLE %s (k int, v vector<int, %%d>, PRIMARY KEY(k, v))",
+ "CREATE TABLE %s (k int, c int, v vector<int, %%d>, PRIMARY KEY(k, c, v))",
+ "CREATE TABLE %s (k int, c int, v vector<int, %%d> static, PRIMARY KEY(k, c))",
+ "CREATE TABLE %s (k int PRIMARY KEY, v vector<int, %%d>)",
// multivalued data types
- testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v list<vector<int, %%d>>)");
- testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v set<vector<int, %%d>>)");
- testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v map<int, vector<int, %%d>>)");
- testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v map<vector<int, %%d>, int>)");
- testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v tuple<vector<int, %%d>, int>)");
- testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v tuple<int, vector<int, %%d>>)");
+ "CREATE TABLE %s (k int PRIMARY KEY, v list<vector<int, %%d>>)",
+ "CREATE TABLE %s (k int PRIMARY KEY, v set<vector<int, %%d>>)",
+ "CREATE TABLE %s (k int PRIMARY KEY, v map<int, vector<int, %%d>>)",
+ "CREATE TABLE %s (k int PRIMARY KEY, v map<vector<int, %%d>, int>)",
+ "CREATE TABLE %s (k int PRIMARY KEY, v tuple<vector<int, %%d>, int>)",
+ "CREATE TABLE %s (k int PRIMARY KEY, v tuple<int, vector<int, %%d>>)",
// nested multivalued data types
- testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v set<frozen<list<vector<int, %%d>>>>)");
- testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v list<frozen<set<vector<int, %%d>>>>)");
- testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v map<int, frozen<set<vector<int, %%d>>>>)");
- testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v map<frozen<set<vector<int, %%d>>>, int>)");
- testCreateTable("CREATE TABLE %s (k int PRIMARY KEY, v tuple<frozen<tuple<vector<int, %%d>, int>>, int>)");
+ "CREATE TABLE %s (k int PRIMARY KEY, v set<frozen<list<vector<int, %%d>>>>)",
+ "CREATE TABLE %s (k int PRIMARY KEY, v list<frozen<set<vector<int, %%d>>>>)",
+ "CREATE TABLE %s (k int PRIMARY KEY, v map<int, frozen<set<vector<int, %%d>>>>)",
+ "CREATE TABLE %s (k int PRIMARY KEY, v map<frozen<set<vector<int, %%d>>>, int>)",
+ "CREATE TABLE %s (k int PRIMARY KEY, v tuple<frozen<tuple<vector<int, %%d>, int>>, int>)"
+ );
+
+ @Test
+ public void testCreateTable() throws Throwable
+ {
+ for (String cql : CREATE_TABLE_CQL)
+ testCreateTable(cql);
}
private void testCreateTable(String query) throws Throwable
@@ -75,16 +81,21 @@ public class GuardrailVectorDimensionsTest extends ThresholdTester
testColumn(() -> format(query, createTableName()));
}
+ static final List<String> CREATE_TYPE_CQL = List.of(
+ "CREATE TYPE %s (c int, v vector<int, %%d>)",
+ "CREATE TYPE %s (c int, v list<vector<int, %%d>>)",
+ "CREATE TYPE %s (c int, v set<vector<int, %%d>>)",
+ "CREATE TYPE %s (c int, v map<int, vector<int, %%d>>)",
+ "CREATE TYPE %s (c int, v map<vector<int, %%d>, int>)",
+ "CREATE TYPE %s (c int, v tuple<vector<int, %%d>, int>)",
+ "CREATE TYPE %s (c int, v tuple<int, vector<int, %%d>>)"
+ );
+
@Test
public void testCreateType() throws Throwable
{
- testCreateType("CREATE TYPE %s (c int, v vector<int, %%d>)");
- testCreateType("CREATE TYPE %s (c int, v list<vector<int, %%d>>)");
- testCreateType("CREATE TYPE %s (c int, v set<vector<int, %%d>>)");
- testCreateType("CREATE TYPE %s (c int, v map<int, vector<int, %%d>>)");
- testCreateType("CREATE TYPE %s (c int, v map<vector<int, %%d>, int>)");
- testCreateType("CREATE TYPE %s (c int, v tuple<vector<int, %%d>, int>)");
- testCreateType("CREATE TYPE %s (c int, v tuple<int, vector<int, %%d>>)");
+ for (String cql : CREATE_TYPE_CQL)
+ testCreateType(cql);;
}
private void testCreateType(String query) throws Throwable
@@ -92,16 +103,21 @@ public class GuardrailVectorDimensionsTest extends ThresholdTester
testField(() -> format(query, createTypeName()));
}
+ static final List<String> ALTER_TABLE_CQL = List.of(
+ "ALTER TABLE %s ADD v vector<int, %%d>",
+ "ALTER TABLE %s ADD v list<vector<int, %%d>>",
+ "ALTER TABLE %s ADD v set<vector<int, %%d>>",
+ "ALTER TABLE %s ADD v map<int, vector<int, %%d>>",
+ "ALTER TABLE %s ADD v map<vector<int, %%d>, int>",
+ "ALTER TABLE %s ADD v tuple<vector<int, %%d>, int>",
+ "ALTER TABLE %s ADD v tuple<int, vector<int, %%d>>"
+ );
+
@Test
public void testAlterTable() throws Throwable
{
- testAlterTable("ALTER TABLE %s ADD v vector<int, %%d>");
- testAlterTable("ALTER TABLE %s ADD v list<vector<int, %%d>>");
- testAlterTable("ALTER TABLE %s ADD v set<vector<int, %%d>>");
- testAlterTable("ALTER TABLE %s ADD v map<int, vector<int, %%d>>");
- testAlterTable("ALTER TABLE %s ADD v map<vector<int, %%d>, int>");
- testAlterTable("ALTER TABLE %s ADD v tuple<vector<int, %%d>, int>");
- testAlterTable("ALTER TABLE %s ADD v tuple<int, vector<int, %%d>>");
+ for (String cql : ALTER_TABLE_CQL)
+ testAlterTable(cql);
}
private void testAlterTable(String query) throws Throwable
@@ -112,16 +128,21 @@ public class GuardrailVectorDimensionsTest extends ThresholdTester
});
}
+ static final List<String> ALTER_TYPE_CQL = List.of(
+ "ALTER TYPE %s ADD v vector<int, %%d>",
+ "ALTER TYPE %s ADD v list<vector<int, %%d>>",
+ "ALTER TYPE %s ADD v set<vector<int, %%d>>",
+ "ALTER TYPE %s ADD v map<int, vector<int, %%d>>",
+ "ALTER TYPE %s ADD v map<vector<int, %%d>, int>",
+ "ALTER TYPE %s ADD v tuple<vector<int, %%d>, int>",
+ "ALTER TYPE %s ADD v tuple<int, vector<int, %%d>>"
+ );
+
@Test
public void testAlterType() throws Throwable
{
- testAlterType("ALTER TYPE %s ADD v vector<int, %%d>");
- testAlterType("ALTER TYPE %s ADD v list<vector<int, %%d>>");
- testAlterType("ALTER TYPE %s ADD v set<vector<int, %%d>>");
- testAlterType("ALTER TYPE %s ADD v map<int, vector<int, %%d>>");
- testAlterType("ALTER TYPE %s ADD v map<vector<int, %%d>, int>");
- testAlterType("ALTER TYPE %s ADD v tuple<vector<int, %%d>, int>");
- testAlterType("ALTER TYPE %s ADD v tuple<int, vector<int, %%d>>");
+ for (String cql : ALTER_TYPE_CQL)
+ testAlterType(cql);
}
private void testAlterType(String query) throws Throwable
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailVectorTypeEnabledTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailVectorTypeEnabledTest.java
new file mode 100644
index 0000000000..38654fa27a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailVectorTypeEnabledTest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.guardrails;
+
+import java.util.List;
+import java.util.function.Supplier;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static java.lang.String.format;
+
+/**
+ * Tests the guardrail for the usage of the vector type, {@link Guardrails#vectorTypeEnabled}.
+ */
+@RunWith(Parameterized.class)
+public class GuardrailVectorTypeEnabledTest extends GuardrailTester
+{
+ private final boolean enabled;
+
+ @Parameterized.Parameters(name = "enabled={0}")
+ public static List<Object[]> params()
+ {
+ return List.of(
+ new Object[]{true},
+ new Object[]{false}
+ );
+ }
+
+ public GuardrailVectorTypeEnabledTest(boolean enabled)
+ {
+ super(Guardrails.vectorTypeEnabled);
+ this.enabled = enabled;
+ Guardrails.instance.setVectorTypeEnabled(enabled);
+ }
+
+ @Test
+ public void testCreateTable() throws Throwable
+ {
+ for (String cql : GuardrailVectorDimensionsTest.CREATE_TABLE_CQL)
+ testCreateTable(cql);
+ }
+
+ private void testCreateTable(String query) throws Throwable
+ {
+ testColumn(() -> format(query, createTableName()));
+ }
+
+ @Test
+ public void testCreateType() throws Throwable
+ {
+ for (String cql : GuardrailVectorDimensionsTest.CREATE_TYPE_CQL)
+ testCreateType(cql);;
+ }
+
+ private void testCreateType(String query) throws Throwable
+ {
+ testField(() -> format(query, createTypeName()));
+ }
+
+ @Test
+ public void testAlterTable() throws Throwable
+ {
+ for (String cql : GuardrailVectorDimensionsTest.ALTER_TABLE_CQL)
+ testAlterTable(cql);
+ }
+
+ private void testAlterTable(String query) throws Throwable
+ {
+ testColumn(() -> {
+ createTable("CREATE TABLE %s (k int PRIMARY KEY)");
+ return format(query, currentTable());
+ });
+ }
+
+ @Test
+ public void testAlterType() throws Throwable
+ {
+ for (String cql : GuardrailVectorDimensionsTest.ALTER_TYPE_CQL)
+ testAlterType(cql);
+ }
+
+ private void testAlterType(String query) throws Throwable
+ {
+ testField(() -> {
+ String name = createType("CREATE TYPE %s (c int)");
+ return format(query, name);
+ });
+ }
+
+ @Test
+ public void testExcludedUsers() throws Throwable
+ {
+ testExcludedUsers(() -> format("CREATE TABLE %s (k int PRIMARY KEY, v vector<int, 1000>)", createTableName()),
+ () -> format("CREATE TYPE %s (c int, v vector<int, 1000>)", createTypeName()),
+ () -> format("ALTER TABLE %s ADD v vector<int, 1000>",
+ createTable("CREATE TABLE %s (k int PRIMARY KEY)")),
+ () -> format("ALTER TYPE %s ADD v vector<int, 1000>",
+ createType("CREATE TYPE %s (c int)")));
+ }
+
+ private void testColumn(Supplier<String> query) throws Throwable
+ {
+ testGuardrail(query, "Column v");
+ }
+
+ private void testField(Supplier<String> query) throws Throwable
+ {
+ testGuardrail(query, "Field v");
+ }
+
+ private void testGuardrail(Supplier<String> query, String what) throws Throwable
+ {
+ if (enabled)
+ assertValid(query.get(), 1);
+ else
+ assertFails(query.get(), what, 1);
+ }
+
+ private void assertValid(String query, int dimensions) throws Throwable
+ {
+ super.assertValid(format(query, dimensions));
+ }
+
+ private void assertFails(String query, String what, int dimensions) throws Throwable
+ {
+ assertFails(format(query, dimensions),
+ what + " is not allowed");
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org