You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ad...@apache.org on 2018/06/25 11:12:19 UTC
[1/3] cassandra git commit: Validate supported column type with SASI
analyzer
Repository: cassandra
Updated Branches:
refs/heads/cassandra-3.11 08a515dc7 -> ea62d8862
refs/heads/trunk 0f5443d9c -> 2bad5d5b6
Validate supported column type with SASI analyzer
patch by Zhao Yang; reviewed by Andres de la Peña for CASSANDRA-13669
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ea62d886
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ea62d886
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ea62d886
Branch: refs/heads/cassandra-3.11
Commit: ea62d8862c311e3d9b64d622bea0a68d3825aa7d
Parents: 08a515d
Author: Zhao Yang <zh...@gmail.com>
Authored: Tue May 15 09:43:39 2018 +0800
Committer: Andrés de la Peña <a....@gmail.com>
Committed: Mon Jun 25 11:51:15 2018 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/index/sasi/SASIIndex.java | 5 +-
.../index/sasi/analyzer/AbstractAnalyzer.java | 8 ++
.../index/sasi/analyzer/DelimiterAnalyzer.java | 12 ++-
.../index/sasi/analyzer/NoOpAnalyzer.java | 6 ++
.../sasi/analyzer/NonTokenizingAnalyzer.java | 6 ++
.../index/sasi/analyzer/StandardAnalyzer.java | 18 +++++
.../cassandra/index/sasi/conf/IndexMode.java | 20 ++++-
.../cassandra/index/sasi/SASIIndexTest.java | 77 ++++++++++++++++++++
9 files changed, 147 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ea62d886/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d9c62ea..6f6e009 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.11.3
+ * Validate supported column type with SASI analyzer (CASSANDRA-13669)
* Remove BTree.Builder Recycler to reduce memory usage (CASSANDRA-13929)
* Reduce nodetool GC thread count (CASSANDRA-14475)
* Fix New SASI view creation during Index Redistribution (CASSANDRA-14055)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ea62d886/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/index/sasi/SASIIndex.java b/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
index f127748..2c1d088 100644
--- a/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
+++ b/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
@@ -121,6 +121,9 @@ public class SASIIndex implements Index, INotificationConsumer
CompactionManager.instance.submitIndexBuild(new SASIIndexBuilder(baseCfs, toRebuild));
}
+ /**
+ * Called via reflection at {@link IndexMetadata#validateCustomIndexOptions}
+ */
public static Map<String, String> validateOptions(Map<String, String> options, CFMetaData cfm)
{
if (!(cfm.partitioner instanceof Murmur3Partitioner))
@@ -140,7 +143,7 @@ public class SASIIndex implements Index, INotificationConsumer
if (target.left.isPartitionKey())
throw new ConfigurationException("partition key columns are not yet supported by SASI");
- IndexMode.validateAnalyzer(options);
+ IndexMode.validateAnalyzer(options, target.left);
IndexMode mode = IndexMode.getMode(target.left, options);
if (mode.mode == Mode.SPARSE)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ea62d886/src/java/org/apache/cassandra/index/sasi/analyzer/AbstractAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/index/sasi/analyzer/AbstractAnalyzer.java b/src/java/org/apache/cassandra/index/sasi/analyzer/AbstractAnalyzer.java
index 31c66cc..e3bb7a2 100644
--- a/src/java/org/apache/cassandra/index/sasi/analyzer/AbstractAnalyzer.java
+++ b/src/java/org/apache/cassandra/index/sasi/analyzer/AbstractAnalyzer.java
@@ -43,6 +43,14 @@ public abstract class AbstractAnalyzer implements Iterator<ByteBuffer>
public abstract void reset(ByteBuffer input);
/**
+ * Test whether the given validator is compatible with the underlying analyzer.
+ *
+ * @param validator
+ * @return
+ */
+ public abstract boolean isCompatibleWith(AbstractType<?> validator);
+
+ /**
* @return true if current analyzer provides text tokenization, false otherwise.
*/
public boolean isTokenizing()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ea62d886/src/java/org/apache/cassandra/index/sasi/analyzer/DelimiterAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/index/sasi/analyzer/DelimiterAnalyzer.java b/src/java/org/apache/cassandra/index/sasi/analyzer/DelimiterAnalyzer.java
index 24acef4..fea4b4f 100644
--- a/src/java/org/apache/cassandra/index/sasi/analyzer/DelimiterAnalyzer.java
+++ b/src/java/org/apache/cassandra/index/sasi/analyzer/DelimiterAnalyzer.java
@@ -37,10 +37,10 @@ import org.apache.cassandra.utils.AbstractIterator;
public class DelimiterAnalyzer extends AbstractAnalyzer
{
- private static final Map<AbstractType<?>,Charset> VALID_ANALYZABLE_TYPES = new HashMap<AbstractType<?>,Charset>()
+ private static final Map<AbstractType<?>, Charset> VALID_ANALYZABLE_TYPES = new HashMap<AbstractType<?>, Charset>()
{{
- put(UTF8Type.instance, StandardCharsets.UTF_8);
- put(AsciiType.instance, StandardCharsets.US_ASCII);
+ put(UTF8Type.instance, StandardCharsets.UTF_8);
+ put(AsciiType.instance, StandardCharsets.US_ASCII);
}};
private char delimiter;
@@ -105,4 +105,10 @@ public class DelimiterAnalyzer extends AbstractAnalyzer
{
return true;
}
+
+ @Override
+ public boolean isCompatibleWith(AbstractType<?> validator)
+ {
+ return VALID_ANALYZABLE_TYPES.containsKey(validator);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ea62d886/src/java/org/apache/cassandra/index/sasi/analyzer/NoOpAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/index/sasi/analyzer/NoOpAnalyzer.java b/src/java/org/apache/cassandra/index/sasi/analyzer/NoOpAnalyzer.java
index 9939a13..5c9b748 100644
--- a/src/java/org/apache/cassandra/index/sasi/analyzer/NoOpAnalyzer.java
+++ b/src/java/org/apache/cassandra/index/sasi/analyzer/NoOpAnalyzer.java
@@ -51,4 +51,10 @@ public class NoOpAnalyzer extends AbstractAnalyzer
this.input = input;
this.hasNext = true;
}
+
+ @Override
+ public boolean isCompatibleWith(AbstractType<?> validator)
+ {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ea62d886/src/java/org/apache/cassandra/index/sasi/analyzer/NonTokenizingAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/index/sasi/analyzer/NonTokenizingAnalyzer.java b/src/java/org/apache/cassandra/index/sasi/analyzer/NonTokenizingAnalyzer.java
index 676b304..82084bc 100644
--- a/src/java/org/apache/cassandra/index/sasi/analyzer/NonTokenizingAnalyzer.java
+++ b/src/java/org/apache/cassandra/index/sasi/analyzer/NonTokenizingAnalyzer.java
@@ -123,4 +123,10 @@ public class NonTokenizingAnalyzer extends AbstractAnalyzer
builder = builder.add("to_lower", new BasicResultFilters.LowerCase());
return builder.build();
}
+
+ @Override
+ public boolean isCompatibleWith(AbstractType<?> validator)
+ {
+ return VALID_ANALYZABLE_TYPES.contains(validator);
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ea62d886/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java b/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java
index 3b58bf9..e1a4a44 100644
--- a/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java
+++ b/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java
@@ -23,10 +23,13 @@ import java.io.InputStreamReader;
import java.io.Reader;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import org.apache.cassandra.index.sasi.analyzer.filter.*;
import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -38,6 +41,15 @@ import com.carrotsearch.hppc.IntObjectOpenHashMap;
public class StandardAnalyzer extends AbstractAnalyzer
{
+
+ private static final Set<AbstractType<?>> VALID_ANALYZABLE_TYPES = new HashSet<AbstractType<?>>()
+ {
+ {
+ add(UTF8Type.instance);
+ add(AsciiType.instance);
+ }
+ };
+
public enum TokenType
{
EOF(-1),
@@ -198,4 +210,10 @@ public class StandardAnalyzer extends AbstractAnalyzer
{
return true;
}
+
+ @Override
+ public boolean isCompatibleWith(AbstractType<?> validator)
+ {
+ return VALID_ANALYZABLE_TYPES.contains(validator);
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ea62d886/src/java/org/apache/cassandra/index/sasi/conf/IndexMode.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/index/sasi/conf/IndexMode.java b/src/java/org/apache/cassandra/index/sasi/conf/IndexMode.java
index c66dd02..5709a0f 100644
--- a/src/java/org/apache/cassandra/index/sasi/conf/IndexMode.java
+++ b/src/java/org/apache/cassandra/index/sasi/conf/IndexMode.java
@@ -93,20 +93,36 @@ public class IndexMode
return analyzer;
}
- public static void validateAnalyzer(Map<String, String> indexOptions) throws ConfigurationException
+ public static void validateAnalyzer(Map<String, String> indexOptions, ColumnDefinition cd) throws ConfigurationException
{
// validate that a valid analyzer class was provided if specified
if (indexOptions.containsKey(INDEX_ANALYZER_CLASS_OPTION))
{
+ Class<?> analyzerClass;
try
{
- Class.forName(indexOptions.get(INDEX_ANALYZER_CLASS_OPTION));
+ analyzerClass = Class.forName(indexOptions.get(INDEX_ANALYZER_CLASS_OPTION));
}
catch (ClassNotFoundException e)
{
throw new ConfigurationException(String.format("Invalid analyzer class option specified [%s]",
indexOptions.get(INDEX_ANALYZER_CLASS_OPTION)));
}
+
+ AbstractAnalyzer analyzer;
+ try
+ {
+ analyzer = (AbstractAnalyzer) analyzerClass.newInstance();
+ if (!analyzer.isCompatibleWith(cd.type))
+ throw new ConfigurationException(String.format("%s does not support type %s",
+ analyzerClass.getSimpleName(),
+ cd.type.asCQL3Type()));
+ }
+ catch (InstantiationException | IllegalAccessException e)
+ {
+ throw new ConfigurationException(String.format("Unable to initialize analyzer class option specified [%s]",
+ analyzerClass.getSimpleName()));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ea62d886/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
index a57af61..1579857 100644
--- a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
@@ -31,10 +31,12 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.config.Schema;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.*;
@@ -55,6 +57,11 @@ import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.sasi.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sasi.analyzer.DelimiterAnalyzer;
+import org.apache.cassandra.index.sasi.analyzer.NoOpAnalyzer;
+import org.apache.cassandra.index.sasi.analyzer.NonTokenizingAnalyzer;
+import org.apache.cassandra.index.sasi.analyzer.StandardAnalyzer;
import org.apache.cassandra.index.sasi.conf.ColumnIndex;
import org.apache.cassandra.index.sasi.disk.OnDiskIndexBuilder;
import org.apache.cassandra.index.sasi.exceptions.TimeQuotaExceededException;
@@ -2411,6 +2418,76 @@ public class SASIIndexTest
Assert.assertEquals(index.searchMemtable(expression).getCount(), 0);
}
+ @Test
+ public void testAnalyzerValidation()
+ {
+ final String TABLE_NAME = "analyzer_validation";
+ QueryProcessor.executeOnceInternal(String.format("CREATE TABLE %s.%s (" +
+ " pk text PRIMARY KEY, " +
+ " ascii_v ascii, " +
+ " bigint_v bigint, " +
+ " blob_v blob, " +
+ " boolean_v boolean, " +
+ " date_v date, " +
+ " decimal_v decimal, " +
+ " double_v double, " +
+ " float_v float, " +
+ " inet_v inet, " +
+ " int_v int, " +
+ " smallint_v smallint, " +
+ " text_v text, " +
+ " time_v time, " +
+ " timestamp_v timestamp, " +
+ " timeuuid_v timeuuid, " +
+ " tinyint_v tinyint, " +
+ " uuid_v uuid, " +
+ " varchar_v varchar, " +
+ " varint_v varint" +
+ ");",
+ KS_NAME,
+ TABLE_NAME));
+
+ Columns regulars = Schema.instance.getCFMetaData(KS_NAME, TABLE_NAME).partitionColumns().regulars;
+ List<String> allColumns = regulars.stream().map(ColumnDefinition::toString).collect(Collectors.toList());
+ List<String> textColumns = Arrays.asList("text_v", "ascii_v", "varchar_v");
+
+ new HashMap<Class<? extends AbstractAnalyzer>, List<String>>()
+ {{
+ put(StandardAnalyzer.class, textColumns);
+ put(NonTokenizingAnalyzer.class, textColumns);
+ put(DelimiterAnalyzer.class, textColumns);
+ put(NoOpAnalyzer.class, allColumns);
+ }}
+ .forEach((analyzer, supportedColumns) -> {
+ for (String column : allColumns)
+ {
+ String query = String.format("CREATE CUSTOM INDEX ON %s.%s(%s) " +
+ "USING 'org.apache.cassandra.index.sasi.SASIIndex' " +
+ "WITH OPTIONS = {'analyzer_class': '%s', 'mode':'PREFIX'};",
+ KS_NAME, TABLE_NAME, column, analyzer.getName());
+
+ if (supportedColumns.contains(column))
+ {
+ QueryProcessor.executeOnceInternal(query);
+ }
+ else
+ {
+ try
+ {
+ QueryProcessor.executeOnceInternal(query);
+ Assert.fail("Expected ConfigurationException");
+ }
+ catch (ConfigurationException e)
+ {
+ // expected
+ Assert.assertTrue("Unexpected error message " + e.getMessage(),
+ e.getMessage().contains("does not support type"));
+ }
+ }
+ }
+ });
+ }
+
private static ColumnFamilyStore loadData(Map<String, Pair<String, Integer>> data, boolean forceFlush)
{
return loadData(data, System.currentTimeMillis(), forceFlush);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[3/3] cassandra git commit: Merge branch 'cassandra-3.11' into trunk
Posted by ad...@apache.org.
Merge branch 'cassandra-3.11' into trunk
# Conflicts:
# src/java/org/apache/cassandra/index/sasi/SASIIndex.java
# test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/2bad5d5b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/2bad5d5b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/2bad5d5b
Branch: refs/heads/trunk
Commit: 2bad5d5b6d2134ecd3db63d02aa2274299d1d748
Parents: 0f5443d ea62d88
Author: Andrés de la Peña <a....@gmail.com>
Authored: Mon Jun 25 12:11:46 2018 +0100
Committer: Andrés de la Peña <a....@gmail.com>
Committed: Mon Jun 25 12:11:46 2018 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/index/sasi/SASIIndex.java | 5 +-
.../index/sasi/analyzer/AbstractAnalyzer.java | 8 ++
.../index/sasi/analyzer/DelimiterAnalyzer.java | 12 ++-
.../index/sasi/analyzer/NoOpAnalyzer.java | 6 ++
.../sasi/analyzer/NonTokenizingAnalyzer.java | 6 ++
.../index/sasi/analyzer/StandardAnalyzer.java | 18 +++++
.../cassandra/index/sasi/conf/IndexMode.java | 20 ++++-
.../cassandra/index/sasi/SASIIndexTest.java | 77 ++++++++++++++++++++
9 files changed, 147 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2bad5d5b/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index ce945df,6f6e009..047689e
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,260 -1,5 +1,261 @@@
+4.0
+ * Add a virtual table to expose active client connections (CASSANDRA-14458)
+ * Clean up and refactor client metrics (CASSANDRA-14524)
+ * Nodetool import row cache invalidation races with adding sstables to tracker (CASSANDRA-14529)
+ * Fix assertions in LWTs after TableMetadata was made immutable (CASSANDRA-14356)
+ * Abort compactions quicker (CASSANDRA-14397)
+ * Support light-weight transactions in cassandra-stress (CASSANDRA-13529)
+ * Make AsyncOneResponse use the correct timeout (CASSANDRA-14509)
+ * Add option to sanity check tombstones on reads/compactions (CASSANDRA-14467)
+ * Add a virtual table to expose all running sstable tasks (CASSANDRA-14457)
+ * Let nodetool import take a list of directories (CASSANDRA-14442)
+ * Avoid unneeded memory allocations / cpu for disabled log levels (CASSANDRA-14488)
+ * Implement virtual keyspace interface (CASSANDRA-7622)
+ * nodetool import cleanup and improvements (CASSANDRA-14417)
+ * Bump jackson version to >= 2.9.5 (CASSANDRA-14427)
+ * Allow nodetool toppartitions without specifying table (CASSANDRA-14360)
+ * Audit logging for database activity (CASSANDRA-12151)
+ * Clean up build artifacts in docs container (CASSANDRA-14432)
+ * Minor network authz improvements (Cassandra-14413)
+ * Automatic sstable upgrades (CASSANDRA-14197)
+ * Replace deprecated junit.framework.Assert usages with org.junit.Assert (CASSANDRA-14431)
+ * Cassandra-stress throws NPE if insert section isn't specified in user profile (CASSSANDRA-14426)
+ * List clients by protocol versions `nodetool clientstats --by-protocol` (CASSANDRA-14335)
+ * Improve LatencyMetrics performance by reducing write path processing (CASSANDRA-14281)
+ * Add network authz (CASSANDRA-13985)
+ * Use the correct IP/Port for Streaming when localAddress is left unbound (CASSANDRA-14389)
+ * nodetool listsnapshots is missing local system keyspace snapshots (CASSANDRA-14381)
+ * Remove StreamCoordinator.streamExecutor thread pool (CASSANDRA-14402)
+ * Rename nodetool --with-port to --print-port to disambiguate from --port (CASSANDRA-14392)
+ * Client TOPOLOGY_CHANGE messages have wrong port. (CASSANDRA-14398)
+ * Add ability to load new SSTables from a separate directory (CASSANDRA-6719)
+ * Eliminate background repair and probablistic read_repair_chance table options
+ (CASSANDRA-13910)
+ * Bind to correct local address in 4.0 streaming (CASSANDRA-14362)
+ * Use standard Amazon naming for datacenter and rack in Ec2Snitch (CASSANDRA-7839)
+ * Fix junit failure for SSTableReaderTest (CASSANDRA-14387)
+ * Abstract write path for pluggable storage (CASSANDRA-14118)
+ * nodetool describecluster should be more informative (CASSANDRA-13853)
+ * Compaction performance improvements (CASSANDRA-14261)
+ * Refactor Pair usage to avoid boxing ints/longs (CASSANDRA-14260)
+ * Add options to nodetool tablestats to sort and limit output (CASSANDRA-13889)
+ * Rename internals to reflect CQL vocabulary (CASSANDRA-14354)
+ * Add support for hybrid MIN(), MAX() speculative retry policies
+ (CASSANDRA-14293, CASSANDRA-14338, CASSANDRA-14352)
+ * Fix some regressions caused by 14058 (CASSANDRA-14353)
+ * Abstract repair for pluggable storage (CASSANDRA-14116)
+ * Add meaningful toString() impls (CASSANDRA-13653)
+ * Add sstableloader option to accept target keyspace name (CASSANDRA-13884)
+ * Move processing of EchoMessage response to gossip stage (CASSANDRA-13713)
+ * Add coordinator write metric per CF (CASSANDRA-14232)
+ * Correct and clarify SSLFactory.getSslContext method and call sites (CASSANDRA-14314)
+ * Handle static and partition deletion properly on ThrottledUnfilteredIterator (CASSANDRA-14315)
+ * NodeTool clientstats should show SSL Cipher (CASSANDRA-14322)
+ * Add ability to specify driver name and version (CASSANDRA-14275)
+ * Abstract streaming for pluggable storage (CASSANDRA-14115)
+ * Forced incremental repairs should promote sstables if they can (CASSANDRA-14294)
+ * Use Murmur3 for validation compactions (CASSANDRA-14002)
+ * Comma at the end of the seed list is interpretated as localhost (CASSANDRA-14285)
+ * Refactor read executor and response resolver, abstract read repair (CASSANDRA-14058)
+ * Add optional startup delay to wait until peers are ready (CASSANDRA-13993)
+ * Add a few options to nodetool verify (CASSANDRA-14201)
+ * CVE-2017-5929 Security vulnerability and redefine default log rotation policy (CASSANDRA-14183)
+ * Use JVM default SSL validation algorithm instead of custom default (CASSANDRA-13259)
+ * Better document in code InetAddressAndPort usage post 7544, incorporate port into UUIDGen node (CASSANDRA-14226)
+ * Fix sstablemetadata date string for minLocalDeletionTime (CASSANDRA-14132)
+ * Make it possible to change neverPurgeTombstones during runtime (CASSANDRA-14214)
+ * Remove GossipDigestSynVerbHandler#doSort() (CASSANDRA-14174)
+ * Add nodetool clientlist (CASSANDRA-13665)
+ * Revert ProtocolVersion changes from CASSANDRA-7544 (CASSANDRA-14211)
+ * Non-disruptive seed node list reload (CASSANDRA-14190)
+ * Nodetool tablehistograms to print statics for all the tables (CASSANDRA-14185)
+ * Migrate dtests to use pytest and python3 (CASSANDRA-14134)
+ * Allow storage port to be configurable per node (CASSANDRA-7544)
+ * Make sub-range selection for non-frozen collections return null instead of empty (CASSANDRA-14182)
+ * BloomFilter serialization format should not change byte ordering (CASSANDRA-9067)
+ * Remove unused on-heap BloomFilter implementation (CASSANDRA-14152)
+ * Delete temp test files on exit (CASSANDRA-14153)
+ * Make PartitionUpdate and Mutation immutable (CASSANDRA-13867)
+ * Fix CommitLogReplayer exception for CDC data (CASSANDRA-14066)
+ * Fix cassandra-stress startup failure (CASSANDRA-14106)
+ * Remove initialDirectories from CFS (CASSANDRA-13928)
+ * Fix trivial log format error (CASSANDRA-14015)
+ * Allow sstabledump to do a json object per partition (CASSANDRA-13848)
+ * Add option to optimise merkle tree comparison across replicas (CASSANDRA-3200)
+ * Remove unused and deprecated methods from AbstractCompactionStrategy (CASSANDRA-14081)
+ * Fix Distribution.average in cassandra-stress (CASSANDRA-14090)
+ * Support a means of logging all queries as they were invoked (CASSANDRA-13983)
+ * Presize collections (CASSANDRA-13760)
+ * Add GroupCommitLogService (CASSANDRA-13530)
+ * Parallelize initial materialized view build (CASSANDRA-12245)
+ * Fix flaky SecondaryIndexManagerTest.assert[Not]MarkedAsBuilt (CASSANDRA-13965)
+ * Make LWTs send resultset metadata on every request (CASSANDRA-13992)
+ * Fix flaky indexWithFailedInitializationIsNotQueryableAfterPartialRebuild (CASSANDRA-13963)
+ * Introduce leaf-only iterator (CASSANDRA-9988)
+ * Upgrade Guava to 23.3 and Airline to 0.8 (CASSANDRA-13997)
+ * Allow only one concurrent call to StatusLogger (CASSANDRA-12182)
+ * Refactoring to specialised functional interfaces (CASSANDRA-13982)
+ * Speculative retry should allow more friendly params (CASSANDRA-13876)
+ * Throw exception if we send/receive repair messages to incompatible nodes (CASSANDRA-13944)
+ * Replace usages of MessageDigest with Guava's Hasher (CASSANDRA-13291)
+ * Add nodetool cmd to print hinted handoff window (CASSANDRA-13728)
+ * Fix some alerts raised by static analysis (CASSANDRA-13799)
+ * Checksum sstable metadata (CASSANDRA-13321, CASSANDRA-13593)
+ * Add result set metadata to prepared statement MD5 hash calculation (CASSANDRA-10786)
+ * Refactor GcCompactionTest to avoid boxing (CASSANDRA-13941)
+ * Expose recent histograms in JmxHistograms (CASSANDRA-13642)
+ * Fix buffer length comparison when decompressing in netty-based streaming (CASSANDRA-13899)
+ * Properly close StreamCompressionInputStream to release any ByteBuf (CASSANDRA-13906)
+ * Add SERIAL and LOCAL_SERIAL support for cassandra-stress (CASSANDRA-13925)
+ * LCS needlessly checks for L0 STCS candidates multiple times (CASSANDRA-12961)
+ * Correctly close netty channels when a stream session ends (CASSANDRA-13905)
+ * Update lz4 to 1.4.0 (CASSANDRA-13741)
+ * Optimize Paxos prepare and propose stage for local requests (CASSANDRA-13862)
+ * Throttle base partitions during MV repair streaming to prevent OOM (CASSANDRA-13299)
+ * Use compaction threshold for STCS in L0 (CASSANDRA-13861)
+ * Fix problem with min_compress_ratio: 1 and disallow ratio < 1 (CASSANDRA-13703)
+ * Add extra information to SASI timeout exception (CASSANDRA-13677)
+ * Add incremental repair support for --hosts, --force, and subrange repair (CASSANDRA-13818)
+ * Rework CompactionStrategyManager.getScanners synchronization (CASSANDRA-13786)
+ * Add additional unit tests for batch behavior, TTLs, Timestamps (CASSANDRA-13846)
+ * Add keyspace and table name in schema validation exception (CASSANDRA-13845)
+ * Emit metrics whenever we hit tombstone failures and warn thresholds (CASSANDRA-13771)
+ * Make netty EventLoopGroups daemon threads (CASSANDRA-13837)
+ * Race condition when closing stream sessions (CASSANDRA-13852)
+ * NettyFactoryTest is failing in trunk on macOS (CASSANDRA-13831)
+ * Allow changing log levels via nodetool for related classes (CASSANDRA-12696)
+ * Add stress profile yaml with LWT (CASSANDRA-7960)
+ * Reduce memory copies and object creations when acting on ByteBufs (CASSANDRA-13789)
+ * Simplify mx4j configuration (Cassandra-13578)
+ * Fix trigger example on 4.0 (CASSANDRA-13796)
+ * Force minumum timeout value (CASSANDRA-9375)
+ * Use netty for streaming (CASSANDRA-12229)
+ * Use netty for internode messaging (CASSANDRA-8457)
+ * Add bytes repaired/unrepaired to nodetool tablestats (CASSANDRA-13774)
+ * Don't delete incremental repair sessions if they still have sstables (CASSANDRA-13758)
+ * Fix pending repair manager index out of bounds check (CASSANDRA-13769)
+ * Don't use RangeFetchMapCalculator when RF=1 (CASSANDRA-13576)
+ * Don't optimise trivial ranges in RangeFetchMapCalculator (CASSANDRA-13664)
+ * Use an ExecutorService for repair commands instead of new Thread(..).start() (CASSANDRA-13594)
+ * Fix race / ref leak in anticompaction (CASSANDRA-13688)
+ * Expose tasks queue length via JMX (CASSANDRA-12758)
+ * Fix race / ref leak in PendingRepairManager (CASSANDRA-13751)
+ * Enable ppc64le runtime as unsupported architecture (CASSANDRA-13615)
+ * Improve sstablemetadata output (CASSANDRA-11483)
+ * Support for migrating legacy users to roles has been dropped (CASSANDRA-13371)
+ * Introduce error metrics for repair (CASSANDRA-13387)
+ * Refactoring to primitive functional interfaces in AuthCache (CASSANDRA-13732)
+ * Update metrics to 3.1.5 (CASSANDRA-13648)
+ * batch_size_warn_threshold_in_kb can now be set at runtime (CASSANDRA-13699)
+ * Avoid always rebuilding secondary indexes at startup (CASSANDRA-13725)
+ * Upgrade JMH from 1.13 to 1.19 (CASSANDRA-13727)
+ * Upgrade SLF4J from 1.7.7 to 1.7.25 (CASSANDRA-12996)
+ * Default for start_native_transport now true if not set in config (CASSANDRA-13656)
+ * Don't add localhost to the graph when calculating where to stream from (CASSANDRA-13583)
+ * Make CDC availability more deterministic via hard-linking (CASSANDRA-12148)
+ * Allow skipping equality-restricted clustering columns in ORDER BY clause (CASSANDRA-10271)
+ * Use common nowInSec for validation compactions (CASSANDRA-13671)
+ * Improve handling of IR prepare failures (CASSANDRA-13672)
+ * Send IR coordinator messages synchronously (CASSANDRA-13673)
+ * Flush system.repair table before IR finalize promise (CASSANDRA-13660)
+ * Fix column filter creation for wildcard queries (CASSANDRA-13650)
+ * Add 'nodetool getbatchlogreplaythrottle' and 'nodetool setbatchlogreplaythrottle' (CASSANDRA-13614)
+ * fix race condition in PendingRepairManager (CASSANDRA-13659)
+ * Allow noop incremental repair state transitions (CASSANDRA-13658)
+ * Run repair with down replicas (CASSANDRA-10446)
+ * Added started & completed repair metrics (CASSANDRA-13598)
+ * Added started & completed repair metrics (CASSANDRA-13598)
+ * Improve secondary index (re)build failure and concurrency handling (CASSANDRA-10130)
+ * Improve calculation of available disk space for compaction (CASSANDRA-13068)
+ * Change the accessibility of RowCacheSerializer for third party row cache plugins (CASSANDRA-13579)
+ * Allow sub-range repairs for a preview of repaired data (CASSANDRA-13570)
+ * NPE in IR cleanup when columnfamily has no sstables (CASSANDRA-13585)
+ * Fix Randomness of stress values (CASSANDRA-12744)
+ * Allow selecting Map values and Set elements (CASSANDRA-7396)
+ * Fast and garbage-free Streaming Histogram (CASSANDRA-13444)
+ * Update repairTime for keyspaces on completion (CASSANDRA-13539)
+ * Add configurable upper bound for validation executor threads (CASSANDRA-13521)
+ * Bring back maxHintTTL propery (CASSANDRA-12982)
+ * Add testing guidelines (CASSANDRA-13497)
+ * Add more repair metrics (CASSANDRA-13531)
+ * RangeStreamer should be smarter when picking endpoints for streaming (CASSANDRA-4650)
+ * Avoid rewrapping an exception thrown for cache load functions (CASSANDRA-13367)
+ * Log time elapsed for each incremental repair phase (CASSANDRA-13498)
+ * Add multiple table operation support to cassandra-stress (CASSANDRA-8780)
+ * Fix incorrect cqlsh results when selecting same columns multiple times (CASSANDRA-13262)
+ * Fix WriteResponseHandlerTest is sensitive to test execution order (CASSANDRA-13421)
+ * Improve incremental repair logging (CASSANDRA-13468)
+ * Start compaction when incremental repair finishes (CASSANDRA-13454)
+ * Add repair streaming preview (CASSANDRA-13257)
+ * Cleanup isIncremental/repairedAt usage (CASSANDRA-13430)
+ * Change protocol to allow sending key space independent of query string (CASSANDRA-10145)
+ * Make gc_log and gc_warn settable at runtime (CASSANDRA-12661)
+ * Take number of files in L0 in account when estimating remaining compaction tasks (CASSANDRA-13354)
+ * Skip building views during base table streams on range movements (CASSANDRA-13065)
+ * Improve error messages for +/- operations on maps and tuples (CASSANDRA-13197)
+ * Remove deprecated repair JMX APIs (CASSANDRA-11530)
+ * Fix version check to enable streaming keep-alive (CASSANDRA-12929)
+ * Make it possible to monitor an ideal consistency level separate from actual consistency level (CASSANDRA-13289)
+ * Outbound TCP connections ignore internode authenticator (CASSANDRA-13324)
+ * Upgrade junit from 4.6 to 4.12 (CASSANDRA-13360)
+ * Cleanup ParentRepairSession after repairs (CASSANDRA-13359)
+ * Upgrade snappy-java to 1.1.2.6 (CASSANDRA-13336)
+ * Incremental repair not streaming correct sstables (CASSANDRA-13328)
+ * Upgrade the jna version to 4.3.0 (CASSANDRA-13300)
+ * Add the currentTimestamp, currentDate, currentTime and currentTimeUUID functions (CASSANDRA-13132)
+ * Remove config option index_interval (CASSANDRA-10671)
+ * Reduce lock contention for collection types and serializers (CASSANDRA-13271)
+ * Make it possible to override MessagingService.Verb ids (CASSANDRA-13283)
+ * Avoid synchronized on prepareForRepair in ActiveRepairService (CASSANDRA-9292)
+ * Adds the ability to use uncompressed chunks in compressed files (CASSANDRA-10520)
+ * Don't flush sstables when streaming for incremental repair (CASSANDRA-13226)
+ * Remove unused method (CASSANDRA-13227)
+ * Fix minor bugs related to #9143 (CASSANDRA-13217)
+ * Output warning if user increases RF (CASSANDRA-13079)
+ * Remove pre-3.0 streaming compatibility code for 4.0 (CASSANDRA-13081)
+ * Add support for + and - operations on dates (CASSANDRA-11936)
+ * Fix consistency of incrementally repaired data (CASSANDRA-9143)
+ * Increase commitlog version (CASSANDRA-13161)
+ * Make TableMetadata immutable, optimize Schema (CASSANDRA-9425)
+ * Refactor ColumnCondition (CASSANDRA-12981)
+ * Parallelize streaming of different keyspaces (CASSANDRA-4663)
+ * Improved compactions metrics (CASSANDRA-13015)
+ * Speed-up start-up sequence by avoiding un-needed flushes (CASSANDRA-13031)
+ * Use Caffeine (W-TinyLFU) for on-heap caches (CASSANDRA-10855)
+ * Thrift removal (CASSANDRA-11115)
+ * Remove pre-3.0 compatibility code for 4.0 (CASSANDRA-12716)
+ * Add column definition kind to dropped columns in schema (CASSANDRA-12705)
+ * Add (automate) Nodetool Documentation (CASSANDRA-12672)
+ * Update bundled cqlsh python driver to 3.7.0 (CASSANDRA-12736)
+ * Reject invalid replication settings when creating or altering a keyspace (CASSANDRA-12681)
+ * Clean up the SSTableReader#getScanner API wrt removal of RateLimiter (CASSANDRA-12422)
+ * Use new token allocation for non bootstrap case as well (CASSANDRA-13080)
+ * Avoid byte-array copy when key cache is disabled (CASSANDRA-13084)
+ * Require forceful decommission if number of nodes is less than replication factor (CASSANDRA-12510)
+ * Allow IN restrictions on column families with collections (CASSANDRA-12654)
+ * Log message size in trace message in OutboundTcpConnection (CASSANDRA-13028)
+ * Add timeUnit Days for cassandra-stress (CASSANDRA-13029)
+ * Add mutation size and batch metrics (CASSANDRA-12649)
+ * Add method to get size of endpoints to TokenMetadata (CASSANDRA-12999)
+ * Expose time spent waiting in thread pool queue (CASSANDRA-8398)
+ * Conditionally update index built status to avoid unnecessary flushes (CASSANDRA-12969)
+ * cqlsh auto completion: refactor definition of compaction strategy options (CASSANDRA-12946)
+ * Add support for arithmetic operators (CASSANDRA-11935)
+ * Add histogram for delay to deliver hints (CASSANDRA-13234)
+ * Fix cqlsh automatic protocol downgrade regression (CASSANDRA-13307)
+ * Changing `max_hint_window_in_ms` at runtime (CASSANDRA-11720)
+ * Trivial format error in StorageProxy (CASSANDRA-13551)
+ * Nodetool repair can hang forever if we lose the notification for the repair completing/failing (CASSANDRA-13480)
+ * Anticompaction can cause noisy log messages (CASSANDRA-13684)
+ * Switch to client init for sstabledump (CASSANDRA-13683)
+ * CQLSH: Don't pause when capturing data (CASSANDRA-13743)
+ * nodetool clearsnapshot requires --all to clear all snapshots (CASSANDRA-13391)
+ * Correctly count range tombstones in traces and tombstone thresholds (CASSANDRA-8527)
+ * cqlshrc.sample uses incorrect option for time formatting (CASSANDRA-14243)
+
+
3.11.3
+ * Validate supported column type with SASI analyzer (CASSANDRA-13669)
* Remove BTree.Builder Recycler to reduce memory usage (CASSANDRA-13929)
* Reduce nodetool GC thread count (CASSANDRA-14475)
* Fix New SASI view creation during Index Redistribution (CASSANDRA-14055)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2bad5d5b/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/index/sasi/SASIIndex.java
index 50b35fd,2c1d088..76e5801
--- a/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
+++ b/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
@@@ -124,9 -121,12 +124,12 @@@ public class SASIIndex implements Index
CompactionManager.instance.submitIndexBuild(new SASIIndexBuilder(baseCfs, toRebuild));
}
+ /**
+ * Called via reflection at {@link IndexMetadata#validateCustomIndexOptions}
+ */
- public static Map<String, String> validateOptions(Map<String, String> options, CFMetaData cfm)
+ public static Map<String, String> validateOptions(Map<String, String> options, TableMetadata metadata)
{
- if (!(cfm.partitioner instanceof Murmur3Partitioner))
+ if (!(metadata.partitioner instanceof Murmur3Partitioner))
throw new ConfigurationException("SASI only supports Murmur3Partitioner.");
String targetColumn = options.get("target");
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2bad5d5b/src/java/org/apache/cassandra/index/sasi/conf/IndexMode.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/index/sasi/conf/IndexMode.java
index d319636,5709a0f..26b18a1
--- a/src/java/org/apache/cassandra/index/sasi/conf/IndexMode.java
+++ b/src/java/org/apache/cassandra/index/sasi/conf/IndexMode.java
@@@ -93,7 -93,7 +93,7 @@@ public class IndexMod
return analyzer;
}
- public static void validateAnalyzer(Map<String, String> indexOptions) throws ConfigurationException
- public static void validateAnalyzer(Map<String, String> indexOptions, ColumnDefinition cd) throws ConfigurationException
++ public static void validateAnalyzer(Map<String, String> indexOptions, ColumnMetadata cd) throws ConfigurationException
{
// validate that a valid analyzer class was provided if specified
if (indexOptions.containsKey(INDEX_ANALYZER_CLASS_OPTION))
http://git-wip-us.apache.org/repos/asf/cassandra/blob/2bad5d5b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
index 515abdf,1579857..c617764
--- a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
@@@ -31,18 -31,18 +31,20 @@@ import java.util.concurrent.Executors
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+ import java.util.stream.Collectors;
import org.apache.cassandra.SchemaLoader;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.ColumnDefinition;
-import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.index.Index;
+import org.apache.cassandra.schema.ColumnMetadata;
++import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.Term;
import org.apache.cassandra.cql3.statements.IndexTarget;
-import org.apache.cassandra.cql3.statements.SelectStatement;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.filter.DataLimits;
@@@ -2409,6 -2418,76 +2416,76 @@@ public class SASIIndexTes
Assert.assertEquals(index.searchMemtable(expression).getCount(), 0);
}
+ @Test
+ public void testAnalyzerValidation()
+ {
+ final String TABLE_NAME = "analyzer_validation";
+ QueryProcessor.executeOnceInternal(String.format("CREATE TABLE %s.%s (" +
+ " pk text PRIMARY KEY, " +
+ " ascii_v ascii, " +
+ " bigint_v bigint, " +
+ " blob_v blob, " +
+ " boolean_v boolean, " +
+ " date_v date, " +
+ " decimal_v decimal, " +
+ " double_v double, " +
+ " float_v float, " +
+ " inet_v inet, " +
+ " int_v int, " +
+ " smallint_v smallint, " +
+ " text_v text, " +
+ " time_v time, " +
+ " timestamp_v timestamp, " +
+ " timeuuid_v timeuuid, " +
+ " tinyint_v tinyint, " +
+ " uuid_v uuid, " +
+ " varchar_v varchar, " +
+ " varint_v varint" +
+ ");",
+ KS_NAME,
+ TABLE_NAME));
+
- Columns regulars = Schema.instance.getCFMetaData(KS_NAME, TABLE_NAME).partitionColumns().regulars;
- List<String> allColumns = regulars.stream().map(ColumnDefinition::toString).collect(Collectors.toList());
++ Columns regulars = Schema.instance.getTableMetadata(KS_NAME, TABLE_NAME).regularColumns();
++ List<String> allColumns = regulars.stream().map(ColumnMetadata::toString).collect(Collectors.toList());
+ List<String> textColumns = Arrays.asList("text_v", "ascii_v", "varchar_v");
+
+ new HashMap<Class<? extends AbstractAnalyzer>, List<String>>()
+ {{
+ put(StandardAnalyzer.class, textColumns);
+ put(NonTokenizingAnalyzer.class, textColumns);
+ put(DelimiterAnalyzer.class, textColumns);
+ put(NoOpAnalyzer.class, allColumns);
+ }}
+ .forEach((analyzer, supportedColumns) -> {
+ for (String column : allColumns)
+ {
+ String query = String.format("CREATE CUSTOM INDEX ON %s.%s(%s) " +
+ "USING 'org.apache.cassandra.index.sasi.SASIIndex' " +
+ "WITH OPTIONS = {'analyzer_class': '%s', 'mode':'PREFIX'};",
+ KS_NAME, TABLE_NAME, column, analyzer.getName());
+
+ if (supportedColumns.contains(column))
+ {
+ QueryProcessor.executeOnceInternal(query);
+ }
+ else
+ {
+ try
+ {
+ QueryProcessor.executeOnceInternal(query);
+ Assert.fail("Expected ConfigurationException");
+ }
+ catch (ConfigurationException e)
+ {
+ // expected
+ Assert.assertTrue("Unexpected error message " + e.getMessage(),
+ e.getMessage().contains("does not support type"));
+ }
+ }
+ }
+ });
+ }
+
private static ColumnFamilyStore loadData(Map<String, Pair<String, Integer>> data, boolean forceFlush)
{
return loadData(data, System.currentTimeMillis(), forceFlush);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[2/3] cassandra git commit: Validate supported column type with SASI
analyzer
Posted by ad...@apache.org.
Validate supported column type with SASI analyzer
patch by Zhao Yang; reviewed by Andres de la Peña for CASSANDRA-13669
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ea62d886
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ea62d886
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ea62d886
Branch: refs/heads/trunk
Commit: ea62d8862c311e3d9b64d622bea0a68d3825aa7d
Parents: 08a515d
Author: Zhao Yang <zh...@gmail.com>
Authored: Tue May 15 09:43:39 2018 +0800
Committer: Andrés de la Peña <a....@gmail.com>
Committed: Mon Jun 25 11:51:15 2018 +0100
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/index/sasi/SASIIndex.java | 5 +-
.../index/sasi/analyzer/AbstractAnalyzer.java | 8 ++
.../index/sasi/analyzer/DelimiterAnalyzer.java | 12 ++-
.../index/sasi/analyzer/NoOpAnalyzer.java | 6 ++
.../sasi/analyzer/NonTokenizingAnalyzer.java | 6 ++
.../index/sasi/analyzer/StandardAnalyzer.java | 18 +++++
.../cassandra/index/sasi/conf/IndexMode.java | 20 ++++-
.../cassandra/index/sasi/SASIIndexTest.java | 77 ++++++++++++++++++++
9 files changed, 147 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ea62d886/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d9c62ea..6f6e009 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.11.3
+ * Validate supported column type with SASI analyzer (CASSANDRA-13669)
* Remove BTree.Builder Recycler to reduce memory usage (CASSANDRA-13929)
* Reduce nodetool GC thread count (CASSANDRA-14475)
* Fix New SASI view creation during Index Redistribution (CASSANDRA-14055)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ea62d886/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/index/sasi/SASIIndex.java b/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
index f127748..2c1d088 100644
--- a/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
+++ b/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
@@ -121,6 +121,9 @@ public class SASIIndex implements Index, INotificationConsumer
CompactionManager.instance.submitIndexBuild(new SASIIndexBuilder(baseCfs, toRebuild));
}
+ /**
+ * Called via reflection at {@link IndexMetadata#validateCustomIndexOptions}
+ */
public static Map<String, String> validateOptions(Map<String, String> options, CFMetaData cfm)
{
if (!(cfm.partitioner instanceof Murmur3Partitioner))
@@ -140,7 +143,7 @@ public class SASIIndex implements Index, INotificationConsumer
if (target.left.isPartitionKey())
throw new ConfigurationException("partition key columns are not yet supported by SASI");
- IndexMode.validateAnalyzer(options);
+ IndexMode.validateAnalyzer(options, target.left);
IndexMode mode = IndexMode.getMode(target.left, options);
if (mode.mode == Mode.SPARSE)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ea62d886/src/java/org/apache/cassandra/index/sasi/analyzer/AbstractAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/index/sasi/analyzer/AbstractAnalyzer.java b/src/java/org/apache/cassandra/index/sasi/analyzer/AbstractAnalyzer.java
index 31c66cc..e3bb7a2 100644
--- a/src/java/org/apache/cassandra/index/sasi/analyzer/AbstractAnalyzer.java
+++ b/src/java/org/apache/cassandra/index/sasi/analyzer/AbstractAnalyzer.java
@@ -43,6 +43,14 @@ public abstract class AbstractAnalyzer implements Iterator<ByteBuffer>
public abstract void reset(ByteBuffer input);
/**
+ * Test whether the given validator is compatible with the underlying analyzer.
+ *
+ * @param validator
+ * @return
+ */
+ public abstract boolean isCompatibleWith(AbstractType<?> validator);
+
+ /**
* @return true if current analyzer provides text tokenization, false otherwise.
*/
public boolean isTokenizing()
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ea62d886/src/java/org/apache/cassandra/index/sasi/analyzer/DelimiterAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/index/sasi/analyzer/DelimiterAnalyzer.java b/src/java/org/apache/cassandra/index/sasi/analyzer/DelimiterAnalyzer.java
index 24acef4..fea4b4f 100644
--- a/src/java/org/apache/cassandra/index/sasi/analyzer/DelimiterAnalyzer.java
+++ b/src/java/org/apache/cassandra/index/sasi/analyzer/DelimiterAnalyzer.java
@@ -37,10 +37,10 @@ import org.apache.cassandra.utils.AbstractIterator;
public class DelimiterAnalyzer extends AbstractAnalyzer
{
- private static final Map<AbstractType<?>,Charset> VALID_ANALYZABLE_TYPES = new HashMap<AbstractType<?>,Charset>()
+ private static final Map<AbstractType<?>, Charset> VALID_ANALYZABLE_TYPES = new HashMap<AbstractType<?>, Charset>()
{{
- put(UTF8Type.instance, StandardCharsets.UTF_8);
- put(AsciiType.instance, StandardCharsets.US_ASCII);
+ put(UTF8Type.instance, StandardCharsets.UTF_8);
+ put(AsciiType.instance, StandardCharsets.US_ASCII);
}};
private char delimiter;
@@ -105,4 +105,10 @@ public class DelimiterAnalyzer extends AbstractAnalyzer
{
return true;
}
+
+ @Override
+ public boolean isCompatibleWith(AbstractType<?> validator)
+ {
+ return VALID_ANALYZABLE_TYPES.containsKey(validator);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ea62d886/src/java/org/apache/cassandra/index/sasi/analyzer/NoOpAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/index/sasi/analyzer/NoOpAnalyzer.java b/src/java/org/apache/cassandra/index/sasi/analyzer/NoOpAnalyzer.java
index 9939a13..5c9b748 100644
--- a/src/java/org/apache/cassandra/index/sasi/analyzer/NoOpAnalyzer.java
+++ b/src/java/org/apache/cassandra/index/sasi/analyzer/NoOpAnalyzer.java
@@ -51,4 +51,10 @@ public class NoOpAnalyzer extends AbstractAnalyzer
this.input = input;
this.hasNext = true;
}
+
+ @Override
+ public boolean isCompatibleWith(AbstractType<?> validator)
+ {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ea62d886/src/java/org/apache/cassandra/index/sasi/analyzer/NonTokenizingAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/index/sasi/analyzer/NonTokenizingAnalyzer.java b/src/java/org/apache/cassandra/index/sasi/analyzer/NonTokenizingAnalyzer.java
index 676b304..82084bc 100644
--- a/src/java/org/apache/cassandra/index/sasi/analyzer/NonTokenizingAnalyzer.java
+++ b/src/java/org/apache/cassandra/index/sasi/analyzer/NonTokenizingAnalyzer.java
@@ -123,4 +123,10 @@ public class NonTokenizingAnalyzer extends AbstractAnalyzer
builder = builder.add("to_lower", new BasicResultFilters.LowerCase());
return builder.build();
}
+
+ @Override
+ public boolean isCompatibleWith(AbstractType<?> validator)
+ {
+ return VALID_ANALYZABLE_TYPES.contains(validator);
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ea62d886/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java b/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java
index 3b58bf9..e1a4a44 100644
--- a/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java
+++ b/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java
@@ -23,10 +23,13 @@ import java.io.InputStreamReader;
import java.io.Reader;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import org.apache.cassandra.index.sasi.analyzer.filter.*;
import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.AsciiType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -38,6 +41,15 @@ import com.carrotsearch.hppc.IntObjectOpenHashMap;
public class StandardAnalyzer extends AbstractAnalyzer
{
+
+ private static final Set<AbstractType<?>> VALID_ANALYZABLE_TYPES = new HashSet<AbstractType<?>>()
+ {
+ {
+ add(UTF8Type.instance);
+ add(AsciiType.instance);
+ }
+ };
+
public enum TokenType
{
EOF(-1),
@@ -198,4 +210,10 @@ public class StandardAnalyzer extends AbstractAnalyzer
{
return true;
}
+
+ @Override
+ public boolean isCompatibleWith(AbstractType<?> validator)
+ {
+ return VALID_ANALYZABLE_TYPES.contains(validator);
+ }
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ea62d886/src/java/org/apache/cassandra/index/sasi/conf/IndexMode.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/index/sasi/conf/IndexMode.java b/src/java/org/apache/cassandra/index/sasi/conf/IndexMode.java
index c66dd02..5709a0f 100644
--- a/src/java/org/apache/cassandra/index/sasi/conf/IndexMode.java
+++ b/src/java/org/apache/cassandra/index/sasi/conf/IndexMode.java
@@ -93,20 +93,36 @@ public class IndexMode
return analyzer;
}
- public static void validateAnalyzer(Map<String, String> indexOptions) throws ConfigurationException
+ public static void validateAnalyzer(Map<String, String> indexOptions, ColumnDefinition cd) throws ConfigurationException
{
// validate that a valid analyzer class was provided if specified
if (indexOptions.containsKey(INDEX_ANALYZER_CLASS_OPTION))
{
+ Class<?> analyzerClass;
try
{
- Class.forName(indexOptions.get(INDEX_ANALYZER_CLASS_OPTION));
+ analyzerClass = Class.forName(indexOptions.get(INDEX_ANALYZER_CLASS_OPTION));
}
catch (ClassNotFoundException e)
{
throw new ConfigurationException(String.format("Invalid analyzer class option specified [%s]",
indexOptions.get(INDEX_ANALYZER_CLASS_OPTION)));
}
+
+ AbstractAnalyzer analyzer;
+ try
+ {
+ analyzer = (AbstractAnalyzer) analyzerClass.newInstance();
+ if (!analyzer.isCompatibleWith(cd.type))
+ throw new ConfigurationException(String.format("%s does not support type %s",
+ analyzerClass.getSimpleName(),
+ cd.type.asCQL3Type()));
+ }
+ catch (InstantiationException | IllegalAccessException e)
+ {
+ throw new ConfigurationException(String.format("Unable to initialize analyzer class option specified [%s]",
+ analyzerClass.getSimpleName()));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/ea62d886/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
index a57af61..1579857 100644
--- a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
@@ -31,10 +31,12 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ColumnDefinition;
+import org.apache.cassandra.config.Schema;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.*;
@@ -55,6 +57,11 @@ import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.sasi.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sasi.analyzer.DelimiterAnalyzer;
+import org.apache.cassandra.index.sasi.analyzer.NoOpAnalyzer;
+import org.apache.cassandra.index.sasi.analyzer.NonTokenizingAnalyzer;
+import org.apache.cassandra.index.sasi.analyzer.StandardAnalyzer;
import org.apache.cassandra.index.sasi.conf.ColumnIndex;
import org.apache.cassandra.index.sasi.disk.OnDiskIndexBuilder;
import org.apache.cassandra.index.sasi.exceptions.TimeQuotaExceededException;
@@ -2411,6 +2418,76 @@ public class SASIIndexTest
Assert.assertEquals(index.searchMemtable(expression).getCount(), 0);
}
+ @Test
+ public void testAnalyzerValidation()
+ {
+ final String TABLE_NAME = "analyzer_validation";
+ QueryProcessor.executeOnceInternal(String.format("CREATE TABLE %s.%s (" +
+ " pk text PRIMARY KEY, " +
+ " ascii_v ascii, " +
+ " bigint_v bigint, " +
+ " blob_v blob, " +
+ " boolean_v boolean, " +
+ " date_v date, " +
+ " decimal_v decimal, " +
+ " double_v double, " +
+ " float_v float, " +
+ " inet_v inet, " +
+ " int_v int, " +
+ " smallint_v smallint, " +
+ " text_v text, " +
+ " time_v time, " +
+ " timestamp_v timestamp, " +
+ " timeuuid_v timeuuid, " +
+ " tinyint_v tinyint, " +
+ " uuid_v uuid, " +
+ " varchar_v varchar, " +
+ " varint_v varint" +
+ ");",
+ KS_NAME,
+ TABLE_NAME));
+
+ Columns regulars = Schema.instance.getCFMetaData(KS_NAME, TABLE_NAME).partitionColumns().regulars;
+ List<String> allColumns = regulars.stream().map(ColumnDefinition::toString).collect(Collectors.toList());
+ List<String> textColumns = Arrays.asList("text_v", "ascii_v", "varchar_v");
+
+ new HashMap<Class<? extends AbstractAnalyzer>, List<String>>()
+ {{
+ put(StandardAnalyzer.class, textColumns);
+ put(NonTokenizingAnalyzer.class, textColumns);
+ put(DelimiterAnalyzer.class, textColumns);
+ put(NoOpAnalyzer.class, allColumns);
+ }}
+ .forEach((analyzer, supportedColumns) -> {
+ for (String column : allColumns)
+ {
+ String query = String.format("CREATE CUSTOM INDEX ON %s.%s(%s) " +
+ "USING 'org.apache.cassandra.index.sasi.SASIIndex' " +
+ "WITH OPTIONS = {'analyzer_class': '%s', 'mode':'PREFIX'};",
+ KS_NAME, TABLE_NAME, column, analyzer.getName());
+
+ if (supportedColumns.contains(column))
+ {
+ QueryProcessor.executeOnceInternal(query);
+ }
+ else
+ {
+ try
+ {
+ QueryProcessor.executeOnceInternal(query);
+ Assert.fail("Expected ConfigurationException");
+ }
+ catch (ConfigurationException e)
+ {
+ // expected
+ Assert.assertTrue("Unexpected error message " + e.getMessage(),
+ e.getMessage().contains("does not support type"));
+ }
+ }
+ }
+ });
+ }
+
private static ColumnFamilyStore loadData(Map<String, Pair<String, Integer>> data, boolean forceFlush)
{
return loadData(data, System.currentTimeMillis(), forceFlush);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org