You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ma...@apache.org on 2023/06/02 18:50:43 UTC

[cassandra] branch cep-7-sai updated (9970cf5684 -> 6ac577308c)

This is an automated email from the ASF dual-hosted git repository.

maedhroz pushed a change to branch cep-7-sai
in repository https://gitbox.apache.org/repos/asf/cassandra.git


    omit 9970cf5684 Unify CQLTester#waitForIndex and SAITester#waitForIndexQueryable
    omit fa85a191c5 Allow CQL queries on multiple indexes without ALLOW FILTERING
    omit b45f47d709 Literal on-disk index and index write path (#9)
    omit 7937878b36 In-memory index implementation with query path
    omit 758b4dc923 Add Index Group Interface for Storage Attached Index
     add f2427a0c86 Increment version to 4.0.10
     add ccaedfa8e7 Merge branch 'cassandra-4.0' into cassandra-4.1
     add 8f7e90dcf3 Merge branch 'cassandra-4.1' into trunk
     add 2fce3025c4 Fix nested selection of reversed collections
     add 5869dc0dac Merge branch 'cassandra-4.0' into cassandra-4.1
     add b7ec523aa9 Merge branch 'cassandra-4.1' into trunk
     add bd49f6ff26 Allow keystore and truststore passwords to be nullable
     add 531b4cde43 Merge branch 'cassandra-4.1' into trunk
     add 08b9471a1e Fix sstable_count metric missing from tablestats json/yaml output
     add 3f67827387 Merge branch 'cassandra-3.11' into cassandra-4.0
     add 2208235ce9 Merge branch 'cassandra-4.0' into cassandra-4.1
     add 976b8395f9 Merge branch 'cassandra-4.1' into trunk
     add cd9bed0aea Deadlock updating sstable metadata if disk boundaries need reloading
     add 4c13df58cb Merge branch 'cassandra-4.0' into cassandra-4.1
     add 8df072e104 Merge branch 'cassandra-4.1' into trunk
     add 33d1c4315c Remove the explicit disabling of UseBiasedLocking as it is the default since jdk15
     add 29ed31542b Improve apidocs and pre-conditions to how upgrade paths are included in jvm-dtest-upgrade (UpgradeTestBase)
     add 8b0dc8ad6b When decommissioning should set Severity to limit traffic
     add 09c1e67598  Incremental repairs fail on mixed IPv4/v6 addresses serializing SyncRequest
     add 1d926e8c6f Merge branch 'cassandra-4.0' into cassandra-4.1
     add b028ac9e87 Merge branch 'cassandra-4.1' into trunk
     add 4f348786bd Do not remove truncated_at entry in system.local while dropping an index
     add 81e78192cb Merge branch 'cassandra-3.0' into cassandra-3.11
     add 5bcee5c06a Merge branch 'cassandra-3.11' into cassandra-4.0
     add e733edb6cc Merge branch 'cassandra-4.0' into cassandra-4.1
     add 28b7fdafa2 Merge branch 'cassandra-4.1' into trunk
     add 1a5302608f Test Failure: HintsDisabledTest.testHintedHandoffDisabled
     add 2e5fb2ea1d Merge branch 'cassandra-4.1' into trunk
     add 0f3a990dd2 Fix the capital P usage in the CQL parser
     add 45938e296e Merge branch 'cassandra-3.11' into cassandra-4.0
     add 30220bc723 Merge branch 'cassandra-4.0' into cassandra-4.1
     add 16643c0078 Merge branch 'cassandra-4.1' into trunk
     add db78e746d7 Do not remove SSTables when cause of FSReadError is OutOfMemoryError while using best_effort disk failure policy
     add c74c9bb037 Merge branch 'cassandra-3.0' into cassandra-3.11
     add c4a305d17a Merge branch 'cassandra-3.11' into cassandra-4.0
     add ba72d90141 Merge branch 'cassandra-4.0' into cassandra-4.1
     add cf48c04c00 Merge branch 'cassandra-4.1' into trunk
     add 6cdcf5e56a Prepare debian changelog for 3.11.15
     add b183e1f0de Merge branch 'cassandra-3.11' into cassandra-4.0
     add 1a72fbd9c6 Merge branch 'cassandra-4.0' into cassandra-4.1
     add 63ab8e0928 Merge branch 'cassandra-4.1' into trunk
     add 998a98eae8 Reorganizing doc directories using new info arch
     add 90d0857d34 fix typos in data modeling and getting started docs
     add 1929550fc0 Merge branch 'cassandra-3.11' into cassandra-4.0
     add cfabddcfb0 Merge branch 'cassandra-4.0' into cassandra-4.1
     add d7314a5988 Merge branch 'cassandra-4.1' into trunk
     add 22329ee0be Deprecate/forbid constructors for Integer, Long, Float, Byte, Double, and Short
     add 73da05f83b Replace usages of json-simple dependency by Jackson
     add 602ffcbf3e fix flaky o.a.c.distributed.test.PaxosRepair2Test.paxosRepairHistoryIsntUpdatedInForcedRepair
     add ccf3789219 Merge branch 'cassandra-4.1' into trunk
     add 65c99bfc42 Improve 'Not enough space for compaction' logging messages
     add 82dc54c3a8 Merge branch 'cassandra-4.0' into cassandra-4.1
     add 5acfde21fe Merge branch 'cassandra-4.1' into trunk
     add e72ec4e828 Add sstablepartitions offline tool to find large partitions in sstables
     add 4a62757624 Suppress CVE-2023-2251
     add aa73bc468a Merge branch 'cassandra-3.0' into cassandra-3.11
     add dba4162666 Merge branch 'cassandra-3.11' into cassandra-4.0
     add b6b71c598e Merge branch 'cassandra-3.0' into cassandra-3.11
     add 682ae0c64c Merge branch 'cassandra-3.11' into cassandra-4.0
     add 462934a84b Merge branch 'cassandra-4.0' into cassandra-4.1
     add 42dac9b950 Merge branch 'cassandra-4.1' into trunk
     add ee5b601ce7 Increment version to 3.11.16
     add f7de633ff8 Merge branch 'cassandra-3.11' into cassandra-4.0
     add 7e4fd3653b Merge branch 'cassandra-4.0' into cassandra-4.1
     add 9948753c84 Merge branch 'cassandra-4.1' into trunk
     add 087cffce63 Prepare debian changelog for 3.0.29
     add 9ab3a82ffa Merge branch 'cassandra-3.0' into cassandra-3.11
     add 3be79b5f0f Merge branch 'cassandra-3.11' into cassandra-4.0
     add 933af9b2db Merge branch 'cassandra-4.0' into cassandra-4.1
     add d22112c293 Merge branch 'cassandra-4.1' into trunk
     add af07ed334a Hotfix: add sstablepartitions to redhat/noboolean/cassandra.spec
     add 4759a7353f CEP-15: (C*) Enhance in-memory FileSystem to work with mmap and support tests to add custom logic
     add ae995eb3d3 NPE when deserializing malformed collections from client
     add 7f4e9bb67b Merge branch 'cassandra-4.0' into cassandra-4.1
     add f54ef5e824 Merge branch 'cassandra-4.1' into trunk
     add f650908648 Moved system properties and envs to CassandraRelevantProperties and CassandraRelevantEnv respectively
     add 75194201f1 Fix quoting in toCqlString methods of UDTs and aggregates
     add d7917a5144 Merge branch 'cassandra-4.0' into cassandra-4.1
     add 290bd0d337 Merge branch 'cassandra-4.1' into trunk
     add 5768e5ae41 Assert successful exit code from sstabledump in org.apache.cassandra.cql3.EmptyValuesTest patch by Daniel Jatnieks; reviewed by Ekaterina Dimitrova and Berenguer Blasi for CASSANDRA-18436
     add 0bcf4a1992 Add keyspace_name column to system_views.clients
     add 785e291900 Increment version to 3.0.30
     add b11021ee68 Merge branch 'cassandra-3.0' into cassandra-3.11
     add 2d40ee9019 Merge branch 'cassandra-3.11' into cassandra-4.0
     add 29ca06ef9d Merge branch 'cassandra-4.0' into cassandra-4.1
     add 6a2d3702ef Merge branch 'cassandra-4.1' into trunk
     add b791644fda Allow internal address to change with reconnecting snitches
     add 67d73469b4 Merge branch 'cassandra-4.0' into cassandra-4.1
     add d4e4159259 Merge branch 'cassandra-4.1' into trunk
     add b8e21fb80a Validate the existence of a datacenter in nodetool rebuild
     add f5c42f428c Merge branch 'cassandra-3.0' into cassandra-3.11
     add 2bb634a339 Merge branch 'cassandra-3.11' into cassandra-4.0
     add 402e2f2f73 Merge branch 'cassandra-4.0' into cassandra-4.1
     add 9ba27f398b Merge branch 'cassandra-4.1' into trunk
     add 2ff1ad4788 Add Timestamp Bound Guardrail (bound user supplied timestamps within a certain range)
     add fe0e04c231 Fix sstable formats configuration
     add 1053e3b475 Partial compaction can resurrect deleted data
     add 446a9d1d01 Merge branch 'cassandra-4.0' into cassandra-4.1
     add 8aaaf21127 Merge branch 'cassandra-4.1' into trunk
     add 3ca94d65d3 Remove unnecessary String.format invocation in QueryProcessor when getting a prepared statement from cache
     add ff820290dd Merge branch 'cassandra-3.11' into cassandra-4.0
     add 621faf7740 Merge branch 'cassandra-4.0' into cassandra-4.1
     add 0352dbe920 Merge branch 'cassandra-4.1' into trunk
     add b828f7ea1b Pass down all contact points to driver for cassandra-stress
     add e1e88e5bc4 Merge branch 'cassandra-3.0' into cassandra-3.11
     add dc6ad3f6b1 Merge branch 'cassandra-3.11' into cassandra-4.0
     add f416a94125 Merge branch 'cassandra-4.0' into cassandra-4.1
     add edd35badaf Merge branch 'cassandra-4.1' into trunk
     add 61333964f4 Add HISTORY command for CQLSH
     add 54528bff20 Report network cache info in nodetool
     add 93efe0ee04 Merge branch 'cassandra-4.0' into cassandra-4.1
     add 2dc1639d4f Merge branch 'cassandra-4.1' into trunk
     add aafb4d1944 Improve nodetool enable{audit,fullquery}log
     add 72fc04d238 Merge branch 'cassandra-4.0' into cassandra-4.1
     add f36fd33e33 Merge branch 'cassandra-4.1' into trunk
     add da77d3f729 Prepare debian changelog for 4.0.10
     add 9e7135c666 Merge branch 'cassandra-4.0' into cassandra-4.1
     add 4f75659654 Merge branch 'cassandra-4.1' into trunk
     add f0699bca74 Ninja fix GetFullQueryLogTest
     add 064d0db0fd Merge branch 'cassandra-4.0' into cassandra-4.1
     add b50c3511ef Merge branch 'cassandra-4.1' into trunk
     add cbe8b39f39 Prepare debian changelog for 4.1.2
     add 61d2193c33 Merge branch 'cassandra-4.1' into trunk
     add b0aa44b27d Add guardrail for partition size and deprecate compaction_large_partition_warning_threshold
     add d2f6b8acb6 ninja-fix FullQueryLoggerTest, requires allow_nodetool_archive_command false to begin with CASSANDRA-18550
     add 4555b94345 ninja-fix FullQueryLoggerTest, requires allow_nodetool_archive_command false to begin with CASSANDRA-18550
     add 22d74c7116 Merge branch 'cassandra-4.0' into cassandra-4.1
     add 01726747cc Merge branch 'cassandra-4.1' into trunk
     add ffe4d85df2 Use WithProperties in try-with-resources to improve properties handling in tests
     add 4d2ee09ffa Make cassandra-stress able to read all credentials from a file
     add 71b79cfbe3 Increment version to 4.0.11 and 4.1.3
     add 2101ce947e Merge branch 'cassandra-4.0' into cassandra-4.1
     add f0abf77e64 Merge branch 'cassandra-4.1' into trunk
     add f16fb6765b Implementation of the trie-indexed SSTable format (BTI) as described in CEP-25.
     add fad1f74570 Rename the byte-comparable translation version to OSS50
     add 18cc821e30 Print header and statistics for cassandra-stress output with arbitrary frequency
     add f66adb0278 Remove unnecessary shuffling of GossipDigests in Gossiper#makeRandomGossipDigest
     add 441285d58e Merge branch 'cassandra-4.0' into cassandra-4.1
     add cccc46cb23 Merge branch 'cassandra-4.1' into trunk
     add 43ec184391 Add support for JMX in the in-jvm dtest framework
     add 57293e8281 Merge branch 'cassandra-3.11' into cassandra-4.0
     add 208a71c008 Merge branch 'cassandra-4.0' into cassandra-4.1
     add 4bcadc6bfb Merge branch 'cassandra-4.1' into trunk
     add b72da02611 Add guardrail for partition tombstones and deprecate compaction_tombstone_warning_threshold
     add 527c0cb3a0 remove dh_python use from Debian packaging
     add 1923e6955a Merge branch 'cassandra-3.0' into cassandra-3.11
     add eb6a45fe6d Merge branch 'cassandra-3.11' into cassandra-4.0
     add 60b26fdfb4 Merge branch 'cassandra-4.0' into cassandra-4.1
     add 91bb1c3b47 Merge branch 'cassandra-4.1' into trunk
     add aa671e8f07 remove dh_python use from Debian packaging
     add f72df5449a Merge branch 'cassandra-2.2' into cassandra-3.0
     add 572db17350 Merge branch 'cassandra-3.0' into cassandra-3.11
     add fb41fb46fc Merge branch 'cassandra-3.11' into cassandra-4.0
     add cdcf9bb5a6 Merge branch 'cassandra-4.0' into cassandra-4.1
     add 6cea0cd448 Merge branch 'cassandra-4.1' into trunk
     new 07ad4d8491 Add Index Group Interface for Storage Attached Index
     new 14ed6a7d73 In-memory index implementation with query path
     new 3914fb6674 Literal on-disk index and index write path (#9)
     new b661711765 Allow CQL queries on multiple indexes without ALLOW FILTERING
     new 26f2c942ab Unify CQLTester#waitForIndex and SAITester#waitForIndexQueryable
     new 6ac577308c post-rebase fixes for the rebase on trunk at fad1f7457032544ab6a7b40c5d38ecb8b25899bb

This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version.  This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:

 * -- * -- B -- O -- O -- O   (9970cf5684)
            \
             N -- N -- N   refs/heads/cep-7-sai (6ac577308c)

You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.

Any revisions marked "omit" are not gone; other references still
refer to them.  Any revisions marked "discard" are gone forever.

The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .build/cassandra-deps-template.xml                 |    4 -
 .build/dependency-check-suppressions.xml           |    1 +
 .build/parent-pom-template.xml                     |    7 +-
 CHANGES.txt                                        |   83 +-
 NEWS.txt                                           |   23 +-
 README.asc                                         |    2 +-
 build.xml                                          |   27 +-
 checkstyle.xml                                     |   63 +-
 checkstyle_test.xml                                |   70 +-
 conf/cassandra.yaml                                |  102 +-
 conf/jvm-server.options                            |    3 -
 conf/jvm11-server.options                          |    9 +
 conf/jvm8-server.options                           |    3 +
 debian/cassandra.install                           |    1 +
 debian/rules                                       |    1 -
 doc/modules/ROOT/pages/index.adoc                  |   16 +-
 doc/modules/cassandra/nav.adoc                     |  141 ++-
 .../cassandra/pages/architecture/index.adoc        |    2 +-
 .../{storage_engine.adoc => storage-engine.adoc}   |    0
 .../cassandra/pages/cql/dynamic_data_masking.adoc  |  178 ----
 .../cassandra/pages/data_modeling/index.adoc       |   11 -
 .../cassandra/pages/{ => developing}/cql/SASI.adoc |    0
 .../pages/{ => developing}/cql/appendices.adoc     |    0
 .../pages/{ => developing}/cql/changes.adoc        |    0
 .../pages/{ => developing}/cql/cql_singlefile.adoc |    0
 .../cassandra/pages/{ => developing}/cql/ddl.adoc  |    0
 .../pages/{ => developing}/cql/definitions.adoc    |    0
 .../cassandra/pages/{ => developing}/cql/dml.adoc  |    0
 .../pages/{ => developing}/cql/functions.adoc      |    0
 .../pages/{ => developing}/cql/index.adoc          |    0
 .../pages/{ => developing}/cql/indexes.adoc        |    0
 .../cassandra/pages/{ => developing}/cql/json.adoc |    0
 .../cassandra/pages/{ => developing}/cql/mvs.adoc  |    0
 .../pages/{ => developing}/cql/operators.adoc      |    0
 .../pages/{ => developing}/cql/security.adoc       |    0
 .../pages/{ => developing}/cql/triggers.adoc       |    0
 .../pages/{ => developing}/cql/types.adoc          |    0
 .../data-modeling/data-modeling_conceptual.adoc}   |    2 +-
 .../data-modeling/data-modeling_logical.adoc}      |    6 +-
 .../data-modeling/data-modeling_physical.adoc}     |    6 +-
 .../data-modeling/data-modeling_queries.adoc}      |    2 +-
 .../data-modeling/data-modeling_rdbms.adoc}        |    2 +-
 .../data-modeling/data-modeling_refining.adoc}     |    2 +-
 .../data-modeling/data-modeling_schema.adoc}       |    0
 .../data-modeling/data-modeling_tools.adoc}        |    0
 .../data-modeling}/images/Figure_1_data_model.jpg  |  Bin
 .../data-modeling}/images/Figure_2_data_model.jpg  |  Bin
 .../images/data_modeling_chebotko_logical.png      |  Bin
 .../images/data_modeling_chebotko_physical.png     |  Bin
 .../images/data_modeling_hotel_bucketing.png       |  Bin
 .../images/data_modeling_hotel_erd.png             |  Bin
 .../images/data_modeling_hotel_logical.png         |  Bin
 .../images/data_modeling_hotel_physical.png        |  Bin
 .../images/data_modeling_hotel_queries.png         |  Bin
 .../images/data_modeling_hotel_relational.png      |  Bin
 .../images/data_modeling_reservation_logical.png   |  Bin
 .../images/data_modeling_reservation_physical.png  |  Bin
 .../pages/developing/data-modeling/index.adoc      |   11 +
 .../data-modeling}/intro.adoc                      |    0
 .../configuring.adoc                               |    0
 .../drivers.adoc                                   |    0
 .../cassandra/pages/getting-started/index.adoc     |   30 +
 .../production.adoc                                |    0
 .../querying.adoc                                  |    2 +-
 .../quickstart.adoc                                |    0
 .../cassandra/pages/getting_started/index.adoc     |   30 -
 .../installing.adoc                                |    4 +-
 .../pages/{ => integrating}/plugins/index.adoc     |    0
 .../configuration/cass_cl_archive_file.adoc        |    0
 .../configuration/cass_env_sh_file.adoc            |    0
 .../configuration/cass_jvm_options_file.adoc       |    0
 .../configuration/cass_logback_xml_file.adoc       |    0
 .../configuration/cass_rackdc_file.adoc            |    0
 .../configuration/cass_topo_file.adoc              |    0
 .../configuration/configuration.adoc               |    0
 .../pages/{ => managing}/configuration/index.adoc  |    0
 .../{ => managing}/operating/audit_logging.adoc    |    0
 .../{ => managing}/operating/auditlogging.adoc     |    0
 .../pages/{ => managing}/operating/backups.adoc    |    0
 .../{ => managing}/operating/bloom_filters.adoc    |    0
 .../{ => managing}/operating/bulk_loading.adoc     |    0
 .../pages/{ => managing}/operating/cdc.adoc        |    0
 .../{ => managing}/operating/compaction/index.adoc |    0
 .../{ => managing}/operating/compaction/lcs.adoc   |    0
 .../{ => managing}/operating/compaction/stcs.adoc  |    0
 .../{ => managing}/operating/compaction/twcs.adoc  |    0
 .../{ => managing}/operating/compression.adoc      |    0
 .../operating/denylisting_partitions.adoc          |    0
 .../pages/{ => managing}/operating/fqllogging.adoc |    0
 .../pages/{ => managing}/operating/hardware.adoc   |    0
 .../pages/{ => managing}/operating/hints.adoc      |    0
 .../pages/{ => managing}/operating/index.adoc      |    0
 .../pages/{ => managing}/operating/metrics.adoc    |    0
 .../{ => managing}/operating/read_repair.adoc      |    0
 .../pages/{ => managing}/operating/repair.adoc     |    0
 .../pages/{ => managing}/operating/security.adoc   |    0
 .../{ => managing}/operating/topo_changes.adoc     |    0
 .../operating/transientreplication.adoc            |    0
 .../{ => managing}/operating/virtualtables.adoc    |    0
 .../{ => managing}/tools/cassandra_stress.adoc     |    6 +
 .../pages/{ => managing}/tools/cqlsh.adoc          |   14 +
 .../pages/{ => managing}/tools/hash_password.adoc  |    0
 .../pages/{ => managing}/tools/index.adoc          |    0
 .../pages/{ => managing}/tools/sstable/index.adoc  |    1 +
 .../{ => managing}/tools/sstable/sstabledump.adoc  |    0
 .../tools/sstable/sstableexpiredblockers.adoc      |    0
 .../tools/sstable/sstablelevelreset.adoc           |    0
 .../tools/sstable/sstableloader.adoc               |    0
 .../tools/sstable/sstablemetadata.adoc             |    0
 .../tools/sstable/sstableofflinerelevel.adoc       |    0
 .../managing/tools/sstable/sstablepartitions.adoc  |  171 ++++
 .../tools/sstable/sstablerepairedset.adoc          |    0
 .../{ => managing}/tools/sstable/sstablescrub.adoc |    0
 .../{ => managing}/tools/sstable/sstablesplit.adoc |    0
 .../tools/sstable/sstableupgrade.adoc              |    0
 .../{ => managing}/tools/sstable/sstableutil.adoc  |    0
 .../tools/sstable/sstableverify.adoc               |    0
 .../pages/{new => managing}/virtualtables.adoc     |    0
 doc/modules/cassandra/pages/new/index.adoc         |    7 +-
 .../cassandra/pages/{ => overview}/faq/index.adoc  |    0
 .../cassandra/pages/overview/terminology.adoc      |   23 +
 .../{getting_started => references}/java11.adoc    |    0
 .../cassandra/pages/tooling/cass-stress-deprecated |    0
 .../cassandra/pages/tooling/generate-tokens        |    0
 .../cassandra/pages/tooling/hash-password-tool     |    0
 .../cassandra/partials/nodetool_and_cqlsh.adoc     |    2 +-
 .../partials/nodetool_and_cqlsh_nobin.adoc         |    2 +-
 .../KubernetesSecretsPEMSslContextFactory.java     |    4 +-
 .../KubernetesSecretsSslContextFactory.java        |   25 +-
 .../KubernetesSecretsPEMSslContextFactoryTest.java |   10 +-
 .../KubernetesSecretsSslContextFactoryTest.java    |   32 +-
 pylib/cqlshlib/cqlshhandling.py                    |   10 +-
 pylib/cqlshlib/cqlshmain.py                        |   65 +-
 pylib/cqlshlib/test/test_cqlsh_completion.py       |    4 +-
 redhat/cassandra.spec                              |    1 +
 redhat/noboolean/cassandra.spec                    |    1 +
 src/antlr/Lexer.g                                  |   21 +-
 src/java/org/apache/cassandra/auth/AuthCache.java  |   20 +-
 .../org/apache/cassandra/auth/AuthKeyspace.java    |    3 +-
 .../cassandra/auth/CassandraRoleManager.java       |   16 +-
 .../apache/cassandra/batchlog/BatchlogManager.java |    3 +-
 .../apache/cassandra/cache/AutoSavingCache.java    |  167 +++-
 .../apache/cassandra/concurrent/SEPExecutor.java   |    2 +-
 .../org/apache/cassandra/concurrent/SEPWorker.java |    3 +-
 .../cassandra/config/CassandraRelevantEnv.java     |   17 +-
 .../config/CassandraRelevantProperties.java        |  778 +++++++++------
 src/java/org/apache/cassandra/config/Config.java   |   45 +-
 .../cassandra/config/DatabaseDescriptor.java       |  270 ++++--
 .../org/apache/cassandra/config/DurationSpec.java  |   58 ++
 .../apache/cassandra/config/EncryptionOptions.java |    9 +-
 .../apache/cassandra/config/GuardrailsOptions.java |  142 ++-
 .../cassandra/config/YamlConfigurationLoader.java  |   11 +-
 .../cql3/CustomPayloadMirroringQueryHandler.java   |    3 +-
 src/java/org/apache/cassandra/cql3/Json.java       |   90 +-
 .../org/apache/cassandra/cql3/QueryProcessor.java  |    3 +-
 .../cassandra/cql3/functions/FromJsonFct.java      |    4 +-
 .../cassandra/cql3/functions/UDAggregate.java      |    4 +-
 .../cql3/functions/UDFExecutorService.java         |    3 +-
 .../cassandra/cql3/selection/ElementsSelector.java |   15 +-
 .../cassandra/cql3/selection/Selectable.java       |   30 +-
 .../apache/cassandra/cql3/selection/Selection.java |    3 +-
 .../cql3/statements/ModificationStatement.java     |   11 +
 .../statements/schema/AlterKeyspaceStatement.java  |    8 +-
 .../statements/schema/CreateViewStatement.java     |    3 +-
 .../apache/cassandra/db/ClusteringComparator.java  |    6 +-
 .../org/apache/cassandra/db/ColumnFamilyStore.java |   29 +-
 src/java/org/apache/cassandra/db/DecoratedKey.java |    2 +-
 src/java/org/apache/cassandra/db/DeletionTime.java |   10 +
 src/java/org/apache/cassandra/db/Directories.java  |   15 +-
 .../db/ExpirationDateOverflowHandling.java         |    9 +-
 src/java/org/apache/cassandra/db/Keyspace.java     |    5 +-
 src/java/org/apache/cassandra/db/ReadCommand.java  |    2 +-
 src/java/org/apache/cassandra/db/StorageHook.java  |    4 +-
 .../db/commitlog/CommitLogDescriptor.java          |    6 +-
 .../cassandra/db/commitlog/CommitLogReplayer.java  |   15 +-
 .../db/commitlog/CommitLogSegmentReader.java       |    5 +-
 .../db/compaction/AbstractCompactionStrategy.java  |    2 +-
 .../db/compaction/CompactionController.java        |   24 +-
 .../cassandra/db/compaction/CompactionLogger.java  |   12 +-
 .../db/compaction/CompactionStrategyManager.java   |    3 +-
 .../cassandra/db/compaction/CompactionTask.java    |   26 +-
 .../db/compaction/LeveledCompactionStrategy.java   |    5 +-
 .../db/compaction/LeveledGenerations.java          |    4 +-
 .../TimeWindowCompactionStrategyOptions.java       |   12 +-
 .../apache/cassandra/db/guardrails/Guardrails.java |  144 ++-
 .../cassandra/db/guardrails/GuardrailsConfig.java  |   69 ++
 .../db/guardrails/GuardrailsConfigProvider.java    |   13 +-
 .../cassandra/db/guardrails/GuardrailsMBean.java   |  109 ++-
 .../org/apache/cassandra/db/lifecycle/LogFile.java |   17 +-
 .../org/apache/cassandra/db/marshal/AsciiType.java |    4 +-
 .../org/apache/cassandra/db/marshal/EmptyType.java |    7 +-
 .../org/apache/cassandra/db/marshal/ListType.java  |    4 +-
 .../org/apache/cassandra/db/marshal/MapType.java   |    6 +-
 .../db/marshal/PartitionerDefinedOrder.java        |    2 +-
 .../org/apache/cassandra/db/marshal/SetType.java   |    4 +-
 .../org/apache/cassandra/db/marshal/TupleType.java |    7 +-
 .../org/apache/cassandra/db/marshal/UTF8Type.java  |    4 +-
 .../org/apache/cassandra/db/marshal/UserType.java  |   18 +-
 .../db/memtable/AbstractShardedMemtable.java       |    9 +-
 .../org/apache/cassandra/db/memtable/Flushing.java |    9 +-
 .../apache/cassandra/db/memtable/TrieMemtable.java |    2 +-
 .../cassandra/db/monitoring/MonitoringTask.java    |   11 +-
 .../cassandra/db/repair/PendingAntiCompaction.java |    5 +-
 .../LazilyInitializedUnfilteredRowIterator.java    |   25 +-
 .../streaming/CassandraCompressedStreamReader.java |    2 +-
 .../CassandraEntireSSTableStreamReader.java        |    2 +-
 .../db/streaming/CassandraOutgoingFile.java        |    3 +-
 .../db/streaming/CassandraStreamHeader.java        |   36 +-
 .../db/streaming/CassandraStreamReader.java        |    6 +-
 .../db/streaming/CassandraStreamReceiver.java      |    4 +-
 .../cassandra/db/streaming/ComponentManifest.java  |   11 +-
 src/java/org/apache/cassandra/db/tries/Trie.java   |    2 +-
 .../org/apache/cassandra/db/view/ViewManager.java  |    4 +-
 .../apache/cassandra/db/virtual/ClientsTable.java  |    5 +-
 .../db/virtual/SystemPropertiesTable.java          |    8 +-
 .../cassandra/dht/ByteOrderedPartitioner.java      |    4 +-
 .../org/apache/cassandra/dht/LocalPartitioner.java |    2 +-
 .../apache/cassandra/dht/Murmur3Partitioner.java   |    2 +-
 .../cassandra/dht/OrderPreservingPartitioner.java  |    4 +-
 .../apache/cassandra/dht/RandomPartitioner.java    |    2 +-
 .../org/apache/cassandra/gms/ApplicationState.java |    2 +-
 .../org/apache/cassandra/gms/EndpointState.java    |    5 +
 .../org/apache/cassandra/gms/FailureDetector.java  |   75 +-
 src/java/org/apache/cassandra/gms/Gossiper.java    |  117 ++-
 .../org/apache/cassandra/gms/VersionedValue.java   |   16 +-
 src/java/org/apache/cassandra/hints/Hint.java      |    3 +-
 .../apache/cassandra/hints/HintsBufferPool.java    |    4 +-
 .../org/apache/cassandra/hints/HintsCatalog.java   |    2 +
 .../apache/cassandra/hints/HintsDescriptor.java    |   37 +-
 .../apache/cassandra/index/IndexStatusManager.java |   18 +-
 .../cassandra/index/SecondaryIndexManager.java     |    3 +-
 .../apache/cassandra/index/sai/QueryContext.java   |    4 +-
 .../index/sai/disk/v1/RowAwarePrimaryKeyMap.java   |    6 +-
 .../index/sai/disk/v1/SSTableIndexWriter.java      |    7 +-
 .../sai/disk/v1/sortedterms/SortedTermsWriter.java |    2 +-
 .../index/sai/disk/v1/trie/TriePrefixSearcher.java |    4 +-
 .../apache/cassandra/index/sai/utils/TypeUtil.java |    6 +-
 .../io/sstable/AbstractRowIndexEntry.java          |    4 +-
 .../io/sstable/AbstractSSTableSimpleWriter.java    |   13 +-
 .../cassandra/io/sstable/CQLSSTableWriter.java     |   22 +-
 .../org/apache/cassandra/io/sstable/Component.java |   14 +-
 .../apache/cassandra/io/sstable/Descriptor.java    |   80 +-
 .../io/sstable/RangeAwareSSTableWriter.java        |    4 +-
 .../org/apache/cassandra/io/sstable/SSTable.java   |    6 +-
 .../cassandra/io/sstable/SSTableHeaderFix.java     |    8 +-
 .../cassandra/io/sstable/SSTableReadsListener.java |    2 +-
 .../io/sstable/SSTableSimpleUnsortedWriter.java    |    4 +-
 .../cassandra/io/sstable/SSTableTxnWriter.java     |    2 +-
 .../org/apache/cassandra/io/sstable/SSTable_API.md |   81 +-
 .../io/sstable/format/AbstractSSTableFormat.java   |   39 +-
 .../cassandra/io/sstable/format/SSTableFormat.java |  123 +--
 .../cassandra/io/sstable/format/SSTableReader.java |   21 +-
 .../io/sstable/format/SSTableReaderWithFilter.java |    2 +-
 .../io/sstable/format/SSTableScanner.java          |  299 ++++++
 .../format/SortedTableReaderLoadingBuilder.java    |   69 ++
 .../io/sstable/format/SortedTableVerifier.java     |  172 +++-
 .../io/sstable/format/SortedTableWriter.java       |   18 +
 .../cassandra/io/sstable/format/TOCComponent.java  |    2 +-
 .../cassandra/io/sstable/format/Version.java       |   23 +-
 .../io/sstable/format/VersionAndType.java          |   94 --
 .../cassandra/io/sstable/format/big/BigFormat.java |   57 +-
 .../format/big/BigFormatPartitionWriter.java       |    6 +-
 .../format/big/BigSSTableReaderLoadingBuilder.java |   34 +-
 .../io/sstable/format/big/BigTableKeyReader.java   |    4 +-
 .../io/sstable/format/big/BigTableReader.java      |   12 +-
 .../io/sstable/format/big/BigTableScanner.java     |  307 ++----
 .../io/sstable/format/big/BigTableVerifier.java    |  244 +----
 .../io/sstable/format/big/IndexState.java          |   14 +-
 .../io/sstable/format/big/RowIndexEntry.java       |   13 +-
 .../cassandra/io/sstable/format/bti/BtiFormat.java |  453 +++++++++
 .../cassandra/io/sstable/format/bti/BtiFormat.md   | 1010 ++++++++++++++++++++
 .../format/bti/BtiFormatPartitionWriter.java       |  125 +++
 .../io/sstable/format/bti/BtiTableReader.java      |  534 +++++++++++
 .../format/bti/BtiTableReaderLoadingBuilder.java   |  208 ++++
 .../io/sstable/format/bti/BtiTableScanner.java     |  140 +++
 .../io/sstable/format/bti/BtiTableScrubber.java    |  315 ++++++
 .../io/sstable/format/bti/BtiTableVerifier.java    |   39 +
 .../io/sstable/format/bti/BtiTableWriter.java      |  430 +++++++++
 .../io/sstable/format/bti/PartitionIndex.java      |  454 +++++++++
 .../sstable/format/bti/PartitionIndexBuilder.java  |  234 +++++
 .../io/sstable/format/bti/PartitionIndexEarly.java |   50 +
 .../io/sstable/format/bti/PartitionIterator.java   |  254 +++++
 .../io/sstable/format/bti/RowIndexReader.java      |  193 ++++
 .../format/bti/RowIndexReverseIterator.java        |   66 ++
 .../io/sstable/format/bti/RowIndexWriter.java      |  123 +++
 .../io/sstable/format/bti/SSTableIterator.java     |  112 +++
 .../format/bti/SSTableReversedIterator.java        |  295 ++++++
 .../io/sstable/format/bti/ScrubIterator.java       |   92 ++
 .../sstable/format/bti/ScrubPartitionIterator.java |   54 ++
 .../io/sstable/format/bti/TrieIndexEntry.java      |  119 +++
 .../sstable/indexsummary/IndexSummaryBuilder.java  |    5 +-
 .../sstable/indexsummary/IndexSummarySupport.java  |    4 +-
 .../io/sstable/keycache/KeyCacheSupport.java       |    8 +-
 .../io/sstable/metadata/StatsMetadata.java         |   52 +-
 .../tries/IncrementalDeepTrieWriterPageAware.java  |  270 ++++--
 .../cassandra/io/tries/IncrementalTrieWriter.java  |    3 +
 .../io/tries/IncrementalTrieWriterPageAware.java   |  111 +--
 .../io/tries/IncrementalTrieWriterSimple.java      |   13 +-
 .../cassandra/io/tries/ReverseValueIterator.java   |  221 +++++
 .../org/apache/cassandra/io/tries/TrieNode.java    |  247 ++---
 .../apache/cassandra/io/tries/ValueIterator.java   |  193 ++--
 src/java/org/apache/cassandra/io/tries/Walker.java |   49 +-
 .../io/util/BufferedDataOutputStreamPlus.java      |    9 +-
 .../apache/cassandra/io/util/DataOutputBuffer.java |    7 +-
 .../apache/cassandra/io/util/DataOutputPlus.java   |    4 +-
 .../cassandra/io/util/DataOutputStreamPlus.java    |    6 +-
 src/java/org/apache/cassandra/io/util/File.java    |   19 +
 .../org/apache/cassandra/io/util/PageAware.java    |   50 +
 .../org/apache/cassandra/io/util/PathUtils.java    |    2 +-
 .../apache/cassandra/io/util/SequentialWriter.java |   24 +-
 .../cassandra/io/util/SimpleChunkReader.java       |    7 +-
 .../org/apache/cassandra/io/util/SizedInts.java    |   88 +-
 .../io/util/TailOverridingRebufferer.java          |    2 +-
 .../cassandra/locator/DynamicEndpointSnitch.java   |   29 +-
 .../locator/ReconnectableSnitchHelper.java         |    8 +-
 .../apache/cassandra/locator/SnitchProperties.java |    6 +-
 .../cassandra/metrics/BufferPoolMetrics.java       |    5 +
 .../DecayingEstimatedHistogramReservoir.java       |    3 +-
 .../apache/cassandra/metrics/KeyspaceMetrics.java  |   13 +-
 .../org/apache/cassandra/metrics/TableMetrics.java |   13 +-
 .../org/apache/cassandra/net/MessagingService.java |    7 +
 .../cassandra/net/OutboundConnectionSettings.java  |    8 +-
 .../apache/cassandra/net/OutboundConnections.java  |    5 +-
 .../org/apache/cassandra/net/SocketFactory.java    |    4 +-
 .../cassandra/repair/consistent/LocalSessions.java |   16 +-
 .../cassandra/repair/messages/SyncRequest.java     |    8 +-
 .../apache/cassandra/schema/CompactionParams.java  |    9 +-
 .../schema/DefaultSchemaUpdateHandler.java         |    2 +-
 .../cassandra/schema/MigrationCoordinator.java     |    5 +-
 .../apache/cassandra/schema/SchemaKeyspace.java    |   10 +-
 .../schema/SchemaUpdateHandlerFactoryProvider.java |   11 +-
 src/java/org/apache/cassandra/schema/TableId.java  |   25 +
 .../security/AbstractSslContextFactory.java        |   10 +-
 .../security/FileBasedSslContextFactory.java       |   34 +-
 .../security/PEMBasedSslContextFactory.java        |    5 -
 .../org/apache/cassandra/security/PEMReader.java   |    3 +-
 .../org/apache/cassandra/security/SSLFactory.java  |    5 +-
 .../serializers/CollectionSerializer.java          |    8 +
 .../cassandra/serializers/ListSerializer.java      |    9 +-
 .../cassandra/serializers/MapSerializer.java       |    8 +-
 .../cassandra/serializers/SetSerializer.java       |    4 +-
 .../cassandra/service/ActiveRepairService.java     |   23 +-
 .../org/apache/cassandra/service/CacheService.java |  227 +++--
 .../apache/cassandra/service/CassandraDaemon.java  |   17 +-
 .../org/apache/cassandra/service/ClientState.java  |    6 +-
 .../cassandra/service/DataResurrectionCheck.java   |    6 +-
 .../cassandra/service/DefaultFSErrorHandler.java   |   24 +-
 .../org/apache/cassandra/service/GCInspector.java  |    8 +-
 .../apache/cassandra/service/LoadBroadcaster.java  |    4 +-
 .../cassandra/service/NativeTransportService.java  |    4 +-
 .../cassandra/service/SSTablesGlobalTracker.java   |   43 +-
 .../SSTablesVersionsInUseChangeNotification.java   |   10 +-
 .../apache/cassandra/service/StartupChecks.java    |    7 +-
 .../org/apache/cassandra/service/StorageProxy.java |   11 +-
 .../apache/cassandra/service/StorageService.java   |  123 ++-
 .../cassandra/service/StorageServiceMBean.java     |   16 +-
 .../org/apache/cassandra/service/paxos/Paxos.java  |    6 +-
 .../cassandra/service/paxos/PaxosCommit.java       |    3 +-
 .../cassandra/service/paxos/PaxosRepair.java       |    3 +-
 .../service/paxos/PaxosRequestCallback.java        |    2 +-
 .../apache/cassandra/service/paxos/PaxosState.java |    3 +-
 .../service/paxos/cleanup/PaxosCleanupSession.java |    3 +-
 .../paxos/uncommitted/PaxosStateTracker.java       |   21 +-
 .../paxos/uncommitted/PaxosUncommittedTracker.java |    9 +-
 .../service/reads/range/RangeCommands.java         |    4 +-
 .../service/reads/repair/BlockingReadRepairs.java  |    6 +-
 .../reads/thresholds/CoordinatorWarnings.java      |    4 +-
 .../service/snapshot/SnapshotManifest.java         |    6 +-
 .../org/apache/cassandra/streaming/StreamHook.java |    4 +-
 .../apache/cassandra/streaming/StreamSession.java  |    3 +-
 .../async/StreamingMultiplexedChannel.java         |    6 +-
 .../org/apache/cassandra/tools/HashPassword.java   |    2 +-
 src/java/org/apache/cassandra/tools/JMXTool.java   |    8 +-
 src/java/org/apache/cassandra/tools/NodeProbe.java |   45 +-
 .../org/apache/cassandra/tools/SSTableExport.java  |    2 +-
 .../cassandra/tools/SSTableMetadataViewer.java     |    4 +-
 .../apache/cassandra/tools/SSTablePartitions.java  |  889 +++++++++++++++++
 .../apache/cassandra/tools/StandaloneScrubber.java |    3 +-
 .../apache/cassandra/tools/StandaloneSplitter.java |    3 +-
 .../apache/cassandra/tools/StandaloneUpgrader.java |    6 +-
 .../apache/cassandra/tools/StandaloneVerifier.java |    3 +-
 src/java/org/apache/cassandra/tools/Util.java      |    3 +-
 .../cassandra/tools/nodetool/EnableAuditLog.java   |    4 +-
 .../tools/nodetool/EnableFullQueryLog.java         |    3 +-
 .../org/apache/cassandra/tools/nodetool/Info.java  |   16 +
 .../tools/nodetool/stats/StatsPrinter.java         |   26 +-
 .../tools/nodetool/stats/TableStatsHolder.java     |    2 +
 .../apache/cassandra/tracing/TraceStateImpl.java   |    4 +-
 src/java/org/apache/cassandra/tracing/Tracing.java |    6 +-
 .../org/apache/cassandra/transport/CBUtil.java     |    5 +-
 .../cassandra/transport/PipelineConfigurator.java  |    3 +-
 .../cassandra/transport/ProtocolVersion.java       |    5 +-
 .../org/apache/cassandra/utils/ByteBufferUtil.java |   41 +-
 .../cassandra/utils/DiagnosticSnapshotService.java |    8 +-
 .../org/apache/cassandra/utils/FBUtilities.java    |   96 +-
 .../org/apache/cassandra/utils/GuidGenerator.java  |    2 +-
 src/java/org/apache/cassandra/utils/IFilter.java   |    7 +
 .../org/apache/cassandra/utils/JMXServerUtils.java |   43 +-
 .../cassandra/utils/JVMStabilityInspector.java     |    4 +-
 src/java/org/apache/cassandra/utils/JsonUtils.java |  211 ++++
 .../org/apache/cassandra/utils/MBeanWrapper.java   |  212 +++-
 .../org/apache/cassandra/utils/MonotonicClock.java |    8 +-
 .../utils/RMIClientSocketFactoryImpl.java          |   62 ++
 .../apache/cassandra/utils/ReflectionUtils.java    |   11 +-
 src/java/org/apache/cassandra/utils/SyncUtil.java  |   17 +-
 src/java/org/apache/cassandra/utils/TimeUUID.java  |    6 +-
 .../org/apache/cassandra/utils/binlog/BinLog.java  |    3 +-
 .../cassandra/utils/binlog/BinLogOptions.java      |    7 +
 .../org/apache/cassandra/utils/btree/BTree.java    |    3 +-
 .../utils/bytecomparable/ByteComparable.java       |    2 +-
 .../utils/bytecomparable/ByteComparable.md         |  370 +++----
 .../utils/concurrent/LightweightRecycler.java      |    8 +-
 .../org/apache/cassandra/utils/concurrent/Ref.java |    3 +-
 .../apache/cassandra/utils/vint/VIntCoding.java    |    2 +-
 ...ssandra.io.sstable.format.SSTableFormat$Factory |    2 +
 .../org/apache/cassandra/utils/LongBTreeTest.java  |   11 +-
 ...em-sslcontextfactory-mismatching-passwords.yaml |  154 +++
 ...ndra-pem-sslcontextfactory-unencryptedkeys.yaml |  148 +++
 test/conf/cassandra.yaml                           |    2 +
 test/conf/trie_memtable.yaml                       |    5 +
 .../legacy_da_clust/da-1-bti-CompressionInfo.db    |  Bin 0 -> 207 bytes
 .../legacy_tables/legacy_da_clust/da-1-bti-Data.db |  Bin 0 -> 8746 bytes
 .../legacy_da_clust/da-1-bti-Digest.crc32          |    1 +
 .../legacy_da_clust/da-1-bti-Filter.db}            |  Bin
 .../legacy_da_clust/da-1-bti-Partitions.db         |  Bin 0 -> 62 bytes
 .../legacy_tables/legacy_da_clust/da-1-bti-Rows.db |  Bin 0 -> 563 bytes
 .../legacy_da_clust/da-1-bti-Statistics.db         |  Bin 0 -> 7304 bytes
 .../legacy_tables/legacy_da_clust/da-1-bti-TOC.txt |    8 +
 .../da-1-bti-CompressionInfo.db                    |  Bin 0 -> 199 bytes
 .../legacy_da_clust_counter/da-1-bti-Data.db       |  Bin 0 -> 7798 bytes
 .../legacy_da_clust_counter/da-1-bti-Digest.crc32  |    1 +
 .../legacy_da_clust_counter/da-1-bti-Filter.db}    |  Bin
 .../legacy_da_clust_counter/da-1-bti-Partitions.db |  Bin 0 -> 62 bytes
 .../legacy_da_clust_counter/da-1-bti-Rows.db       |  Bin 0 -> 563 bytes
 .../legacy_da_clust_counter/da-1-bti-Statistics.db |  Bin 0 -> 7313 bytes
 .../legacy_da_clust_counter/da-1-bti-TOC.txt       |    8 +
 .../legacy_da_simple/da-1-bti-CompressionInfo.db}  |  Bin
 .../legacy_da_simple/da-1-bti-Data.db              |  Bin 0 -> 89 bytes
 .../legacy_da_simple/da-1-bti-Digest.crc32         |    1 +
 .../legacy_da_simple/da-1-bti-Filter.db}           |  Bin
 .../legacy_da_simple/da-1-bti-Partitions.db        |  Bin 0 -> 59 bytes
 .../legacy_da_simple/da-1-bti-Rows.db}             |    0
 .../legacy_da_simple/da-1-bti-Statistics.db        |  Bin 0 -> 4814 bytes
 .../legacy_da_simple/da-1-bti-TOC.txt              |    8 +
 .../da-1-bti-CompressionInfo.db}                   |  Bin
 .../legacy_da_simple_counter/da-1-bti-Data.db      |  Bin 0 -> 141 bytes
 .../legacy_da_simple_counter/da-1-bti-Digest.crc32 |    1 +
 .../legacy_da_simple_counter/da-1-bti-Filter.db}   |  Bin
 .../da-1-bti-Partitions.db                         |  Bin 0 -> 60 bytes
 .../legacy_da_simple_counter/da-1-bti-Rows.db}     |    0
 .../da-1-bti-Statistics.db                         |  Bin 0 -> 4823 bytes
 .../legacy_da_simple_counter/da-1-bti-TOC.txt      |    8 +
 .../cassandra/distributed/action/GossipHelper.java |   14 +-
 .../org/apache/cassandra/distributed/api/Row.java  |    2 +-
 .../cassandra/distributed/fuzz/HarryHelper.java    |   27 +-
 .../distributed/impl/AbstractCluster.java          |   16 +-
 .../impl/CollectingRMIServerSocketFactoryImpl.java |   88 ++
 .../distributed/impl/INodeProvisionStrategy.java   |   11 +
 .../cassandra/distributed/impl/Instance.java       |   51 +-
 .../cassandra/distributed/impl/InstanceConfig.java |   16 +-
 .../cassandra/distributed/impl/IsolatedJmx.java    |  230 +++++
 .../cassandra/distributed/impl/TracingUtil.java    |   11 -
 .../cassandra/distributed/shared/Byteman.java      |    4 +-
 .../cassandra/distributed/shared/ClusterUtils.java |   25 +-
 .../distributed/shared/WithProperties.java         |   71 +-
 .../cassandra/distributed/test/AlterTest.java      |    3 +-
 .../test/BootstrapBinaryDisabledTest.java          |   17 +-
 .../distributed/test/CASContentionTest.java        |    3 +-
 .../apache/cassandra/distributed/test/CASTest.java |    3 +-
 .../test/CompactionOverlappingSSTableTest.java     |  117 +++
 .../test/DataResurrectionCheckTest.java            |   13 +-
 .../test/FailingResponseDoesNotLogTest.java        |   19 +-
 .../distributed/test/FailingTruncationTest.java    |    7 +-
 .../cassandra/distributed/test/GossipTest.java     |    3 +-
 .../distributed/test/HintsDisabledTest.java        |   29 +-
 .../distributed/test/IPMembershipTest.java         |    4 +-
 .../distributed/test/IndexDroppingTest.java        |  122 +++
 .../distributed/test/MessageForwardingTest.java    |   13 +-
 .../distributed/test/MigrationCoordinatorTest.java |   16 +-
 .../cassandra/distributed/test/MoveTest.java       |    3 +-
 .../distributed/test/NativeMixedVersionTest.java   |   15 +-
 .../distributed/test/NativeProtocolTest.java       |    3 +-
 .../distributed/test/PartitionDenylistTest.java    |    9 +-
 .../distributed/test/PaxosRepair2Test.java         |  268 +++---
 .../distributed/test/PaxosRepairTest.java          |    2 +-
 .../cassandra/distributed/test/ReadRepairTest.java |    4 +-
 .../distributed/test/RepairCoordinatorBase.java    |    4 +-
 .../distributed/test/ResourceLeakTest.java         |   62 +-
 .../cassandra/distributed/test/TestBaseImpl.java   |   20 +-
 .../distributed/test/TimeoutAbortTest.java         |    3 +-
 .../test/UpdateSystemAuthAfterDCExpansionTest.java |    4 +-
 .../distributed/test/VirtualTableLogsTest.java     |   33 +-
 .../guardrails/GuardrailPartitionSizeTest.java     |  153 +++
 .../GuardrailPartitionTombstonesTest.java          |  142 +++
 .../test/hostreplacement/FailedBootstrapTest.java  |    3 +-
 .../distributed/test/jmx/JMXFeatureTest.java       |  113 +++
 .../distributed/test/jmx/JMXGetterCheckTest.java   |   26 +-
 .../distributed/test/metric/TableMetricTest.java   |   33 +-
 .../distributed/test/ring/BootstrapTest.java       |   27 +-
 .../distributed/test/ring/CleanupFailureTest.java  |    3 +-
 .../distributed/test/ring/PendingWritesTest.java   |    4 +-
 .../test/thresholds/RowIndexSizeWarningTest.java   |    4 +-
 .../DecommissionAvoidReadTimeoutsTest.java}        |   18 +-
 .../test/topology/DecommissionAvoidTimeouts.java   |  238 +++++
 .../DecommissionAvoidWriteTimeoutsTest.java}       |   18 +-
 .../distributed/upgrade/UpgradeTestBase.java       |   14 +-
 .../cassandra/distributed/util/Coordinators.java   |   71 ++
 .../distributed/util/QueryResultUtil.java          |  109 +++
 .../apache/cassandra/distributed/util/TwoWay.java  |   81 ++
 .../util/byterewrite/StatusChangeListener.java     |  133 +++
 .../distributed/util/byterewrite/Undead.java       |   63 ++
 .../io/sstable/format/ForwardingSSTableReader.java |    8 +-
 .../LongLeveledCompactionStrategyCQLTest.java      |    4 +-
 .../db/compaction/CompactionAllocationTest.java    |    2 +-
 .../AbstractTypeByteSourceDecodingBench.java       |    4 +-
 .../test/microbench/CacheLoaderBench.java          |   61 +-
 .../microbench/ZeroCopyStreamingBenchmark.java     |    4 +-
 .../test/microbench/btree/BTreeTransformBench.java |    2 +-
 .../test/microbench/btree/BTreeUpdateBench.java    |    2 +-
 .../test/microbench/tries/ComparisonReadBench.java |   14 +-
 .../cassandra/simulator/asm/InterceptAgent.java    |    2 +
 .../cassandra/simulator/asm/MethodLogger.java      |    2 +
 .../apache/cassandra/simulator/ActionSchedule.java |    3 +-
 .../cassandra/simulator/ClusterSimulation.java     |   47 +-
 .../cassandra/simulator/SimulationRunner.java      |   17 +-
 .../cassandra/simulator/paxos/PaxosSimulation.java |   10 +-
 .../cassandra/AbstractSerializationsTester.java    |    7 +-
 .../CassandraBriefJUnitResultFormatter.java        |    7 +-
 .../CassandraXMLJUnitResultFormatter.java          |    9 +-
 .../apache/cassandra/LogbackStatusListener.java    |   12 +-
 test/unit/org/apache/cassandra/SchemaLoader.java   |   11 +-
 test/unit/org/apache/cassandra/Util.java           |    3 +-
 .../cassandra/audit/AuditLoggerAuthTest.java       |    3 +-
 .../cassandra/audit/AuditLoggerCleanupTest.java    |    4 +-
 .../apache/cassandra/audit/AuditLoggerTest.java    |   29 +
 .../org/apache/cassandra/auth/AuthCacheTest.java   |    6 +-
 .../cassandra/auth/CreateAndAlterRoleTest.java     |    3 +
 .../apache/cassandra/auth/GrantAndRevokeTest.java  |    2 +
 .../cassandra/auth/PasswordAuthenticatorTest.java  |   17 +-
 .../org/apache/cassandra/auth/jmx/JMXAuthTest.java |   12 +-
 .../cassandra/cache/AutoSavingCacheTest.java       |    3 +-
 .../config/CassandraRelevantPropertiesTest.java    |  111 ++-
 .../cassandra/config/ConfigCompatabilityTest.java  |    4 +-
 .../config/DatabaseDescriptorRefTest.java          |    7 +-
 .../cassandra/config/DatabaseDescriptorTest.java   |   37 +-
 .../apache/cassandra/config/DurationSpecTest.java  |    6 +
 .../LoadOldYAMLBackwardCompatibilityTest.java      |    3 +-
 .../config/OverrideConfigurationLoader.java        |    4 +-
 .../config/YamlConfigurationLoaderTest.java        |   18 +-
 .../apache/cassandra/cql3/CDCStatementTest.java    |    2 +
 test/unit/org/apache/cassandra/cql3/CQLTester.java |  166 +++-
 .../cassandra/cql3/CustomNowInSecondsTest.java     |    3 +
 .../cassandra/cql3/DecoratedKeyPrefixesTest.java   |  265 +++++
 .../org/apache/cassandra/cql3/EmptyValuesTest.java |    8 +-
 .../apache/cassandra/cql3/GcCompactionTest.java    |    2 +-
 .../org/apache/cassandra/cql3/KeyCacheCqlTest.java |    7 +-
 .../unit/org/apache/cassandra/cql3/PagingTest.java |    3 +-
 .../cql3/QueryWithIndexedSSTableTest.java          |   36 +-
 .../cql3/TombstonesWithIndexedSSTableTest.java     |   82 +-
 .../apache/cassandra/cql3/ViewFiltering1Test.java  |    6 +-
 .../cql3/selection/SelectionColumnMappingTest.java |    3 +
 .../cql3/statements/DescribeStatementTest.java     |  130 +++
 .../cql3/validation/entities/CollectionsTest.java  |   44 +-
 .../validation/entities/FrozenCollectionsTest.java |    2 +
 .../cql3/validation/entities/JsonTest.java         |   12 +-
 .../cql3/validation/entities/UFSecurityTest.java   |    4 +-
 .../cql3/validation/entities/UserTypesTest.java    |    3 +
 .../cql3/validation/entities/VirtualTableTest.java |    3 +
 .../miscellaneous/SSTablesIteratedTest.java        |    8 +-
 .../cql3/validation/operations/CreateTest.java     |   17 +
 .../cql3/validation/operations/DropTest.java       |    9 +
 .../cql3/validation/operations/InsertTest.java     |   38 +
 .../InsertUpdateIfConditionCollectionsTest.java    |    3 +
 .../InsertUpdateIfConditionStaticsTest.java        |    2 +
 .../operations/InsertUpdateIfConditionTest.java    |    2 +
 .../validation/operations/SelectLimitTest.java     |    3 +
 .../cql3/validation/operations/TTLTest.java        |   10 +-
 .../apache/cassandra/db/ColumnFamilyStoreTest.java |    2 +-
 .../cassandra/db/CorruptPrimaryIndexTest.java      |   96 ++
 .../org/apache/cassandra/db/DirectoriesTest.java   |  141 ++-
 .../unit/org/apache/cassandra/db/KeyspaceTest.java |    4 +-
 .../unit/org/apache/cassandra/db/RowCacheTest.java |    3 +-
 .../apache/cassandra/db/SchemaCQLHelperTest.java   |   11 +-
 .../cassandra/db/SerializationHeaderTest.java      |    4 +-
 .../columniterator/SSTableReverseIteratorTest.java |    2 +-
 .../db/commitlog/CommitLogFailurePolicyTest.java   |    4 +-
 .../commitlog/CommitLogSegmentManagerCDCTest.java  |    3 +
 .../cassandra/db/commitlog/CommitLogTest.java      |  190 ++--
 .../db/compaction/CompactionIteratorTest.java      |    3 +-
 .../db/compaction/CompactionsCQLTest.java          |   12 +-
 .../cassandra/db/compaction/CompactionsTest.java   |   17 +-
 .../compaction/LeveledCompactionStrategyTest.java  |    2 +-
 .../cassandra/db/compaction/NeverPurgeTest.java    |    3 +-
 .../db/compaction/PartialCompactionsTest.java      |  233 +++++
 .../SizeTieredCompactionStrategyTest.java          |   13 +-
 .../cassandra/db/compaction/TTLExpiryTest.java     |    2 +-
 .../TimeWindowCompactionStrategyTest.java          |    6 +-
 .../db/guardrails/GuardrailCollectionSizeTest.java |    6 +-
 .../guardrails/GuardrailColumnValueSizeTest.java   |    6 +-
 .../guardrails/GuardrailMaximumTimestampTest.java  |   74 ++
 .../guardrails/GuardrailMinimumTimestampTest.java  |   74 ++
 .../db/guardrails/GuardrailPartitionSizeTest.java  |  123 +++
 .../GuardrailPartitionTombstonesTest.java          |  127 +++
 .../cassandra/db/guardrails/ThresholdTester.java   |   23 +-
 .../apache/cassandra/db/lifecycle/HelpersTest.java |    8 +-
 .../cassandra/db/lifecycle/LogTransactionTest.java |   54 +-
 .../apache/cassandra/db/lifecycle/TrackerTest.java |    3 +-
 .../cassandra/db/marshal/JsonConversionTest.java   |    6 +-
 .../apache/cassandra/db/marshal/TimeTypeTest.java  |    7 +-
 .../db/memtable/MemtableSizeTestBase.java          |    2 +
 .../apache/cassandra/db/rows/RowsMergingTest.java  |    4 +-
 .../db/rows/ThrottledUnfilteredIteratorTest.java   |    2 +-
 .../CassandraEntireSSTableStreamWriterTest.java    |    9 +-
 .../db/streaming/CassandraStreamHeaderTest.java    |    9 +-
 .../db/streaming/ComponentManifestTest.java        |    4 +-
 .../db/transform/DuplicateRowCheckerTest.java      |    3 +-
 .../apache/cassandra/db/tries/SlicedTrieTest.java  |   12 +-
 .../apache/cassandra/db/tries/TrieToMermaid.java   |  122 +++
 .../tries/TrieToMermaidTest.java}                  |   33 +-
 .../db/virtual/ClientsTableKeyspaceColTest.java    |  132 +++
 .../db/virtual/CredentialsCacheKeysTableTest.java  |    3 +
 .../cassandra/db/virtual/LogMessagesTableTest.java |   10 +-
 .../NetworkPermissionsCacheKeysTableTest.java      |    3 +
 .../db/virtual/PermissionsCacheKeysTableTest.java  |    2 +
 .../db/virtual/RolesCacheKeysTableTest.java        |    2 +
 .../db/virtual/SystemPropertiesTableTest.java      |    4 +-
 .../apache/cassandra/dht/LengthPartitioner.java    |   10 +-
 .../diag/store/DiagnosticEventMemoryStoreTest.java |   22 +-
 .../apache/cassandra/fql/FullQueryLoggerTest.java  |   29 +
 .../apache/cassandra/gms/FailureDetectorTest.java  |    3 +-
 .../org/apache/cassandra/gms/GossiperTest.java     |    3 +-
 .../gms/PendingRangeCalculatorServiceTest.java     |    3 +-
 .../org/apache/cassandra/gms/ShadowRoundTest.java  |   17 +-
 .../apache/cassandra/hints/HintWriteTTLTest.java   |    4 +-
 .../org/apache/cassandra/index/sai/SAITester.java  |   13 +-
 .../sai/disk/v1/InvertedIndexSearcherTest.java     |    2 +-
 .../index/sai/disk/v1/SegmentFlushTest.java        |    2 +-
 .../index/sai/disk/v1/TermsReaderTest.java         |    4 +-
 .../cassandra/index/sai/disk/v1/TermsScanner.java  |    4 +-
 .../sai/disk/v1/sortedterms/SortedTermsTest.java   |   16 +-
 .../sai/disk/v1/trie/TriePrefixSearcherTest.java   |   46 +-
 .../sai/disk/v1/trie/TrieTermsDictionaryTest.java  |    8 +-
 .../index/sai/memory/RAMStringIndexerTest.java     |    2 +-
 .../index/sai/memory/TrieMemoryIndexTest.java      |    2 +-
 .../cassandra/index/sai/plan/OperationTest.java    |    3 +-
 .../index/sai/utils/AbstractPrimaryKeyTester.java  |    2 +-
 .../cassandra/index/sai/utils/TypeUtilTest.java    |    2 +-
 .../index/sai/virtual/SegmentsSystemViewTest.java  |    4 +-
 .../apache/cassandra/index/sasi/SASIIndexTest.java |    7 +-
 .../index/sasi/disk/PerSSTableIndexWriterTest.java |    4 +-
 .../cassandra/index/sasi/plan/OperationTest.java   |    4 +-
 .../apache/cassandra/io/DiskSpaceMetricsTest.java  |    4 +-
 .../io/filesystem/ForwardingFileChannel.java       |  146 +++
 .../io/filesystem/ForwardingFileSystem.java        |  129 +++
 .../filesystem/ForwardingFileSystemProvider.java   |  246 +++++
 .../cassandra/io/filesystem/ForwardingPath.java    |  234 +++++
 .../io/filesystem/ListenableFileSystem.java        |  856 +++++++++++++++++
 .../apache/cassandra/io/sstable/ComponentTest.java |   50 +-
 .../cassandra/io/sstable/DescriptorTest.java       |   17 +-
 .../cassandra/io/sstable/LegacySSTableTest.java    |   19 +-
 .../io/sstable/RangeAwareSSTableWriterTest.java    |    3 +-
 .../io/sstable/SSTableFlushObserverTest.java       |    7 +-
 .../cassandra/io/sstable/SSTableFormatTest.java    |  276 ++++++
 .../io/sstable/SSTableFormatTypeTest.java          |  198 ----
 .../cassandra/io/sstable/SSTableHeaderFixTest.java |    4 +-
 .../cassandra/io/sstable/SSTableReaderTest.java    |  335 ++++++-
 .../cassandra/io/sstable/SSTableScannerTest.java   |  137 ++-
 .../apache/cassandra/io/sstable/SSTableUtils.java  |    2 +-
 .../io/sstable/SSTableWriterTransactionTest.java   |   12 +-
 .../org/apache/cassandra/io/sstable/ScrubTest.java |   45 +-
 .../apache/cassandra/io/sstable/VerifyTest.java    |   16 +-
 .../AbstractTestVersionSupportedFeatures.java      |   91 ++
 .../io/sstable/format/VersionAndTypeTest.java      |   62 --
 .../io/sstable/format/big/RowIndexEntryTest.java   |   10 +-
 .../format/big/VersionSupportedFeaturesTest.java   |   69 ++
 .../io/sstable/format/bti/PartitionIndexTest.java  |  888 +++++++++++++++++
 .../io/sstable/format/bti/RowIndexTest.java        |  517 ++++++++++
 .../format/bti/VersionSupportedFeaturesTest.java   |   70 ++
 .../indexsummary/IndexSummaryManagerTest.java      |    3 +-
 .../IndexSummaryRedistributionTest.java            |    3 +-
 .../io/sstable/indexsummary/IndexSummaryTest.java  |    7 +-
 .../io/sstable/keycache/KeyCacheTest.java          |   15 +-
 .../sstable/metadata/MetadataSerializerTest.java   |   85 +-
 .../cassandra/io/tries/AbstractTrieTestBase.java   |   71 +-
 .../apache/cassandra/io/tries/TrieBuilderTest.java |    8 +-
 .../apache/cassandra/io/tries/TrieNodeTest.java    |   86 +-
 .../org/apache/cassandra/io/tries/WalkerTest.java  |  188 ++--
 .../io/util/BufferedDataOutputStreamTest.java      |    2 +-
 .../apache/cassandra/io/util/DataOutputTest.java   |    2 +-
 .../org/apache/cassandra/io/util/FileSystems.java  |   89 ++
 .../apache/cassandra/io/util/PageAwareTest.java    |   67 +-
 .../apache/cassandra/io/util/SizedIntsTest.java    |   20 +-
 .../io/util/TailOverridingRebuffererTest.java      |   17 +-
 .../cassandra/locator/AlibabaCloudSnitchTest.java  |    3 +-
 .../cassandra/locator/CloudstackSnitchTest.java    |    3 +-
 .../apache/cassandra/locator/EC2SnitchTest.java    |    3 +-
 .../cassandra/locator/GoogleCloudSnitchTest.java   |    3 +-
 .../cassandra/locator/PropertyFileSnitchTest.java  |    3 +-
 .../cassandra/metrics/TrieMemtableMetricsTest.java |    4 +-
 .../org/apache/cassandra/net/ConnectionTest.java   |   12 +-
 .../net/MessageSerializationPropertyTest.java      |    9 +-
 .../org/apache/cassandra/schema/MockSchema.java    |   96 +-
 .../schema/RemoveWithoutDroppingTest.java          |    6 +-
 .../CustomSslContextFactoryConfigTest.java         |    9 +-
 .../CustomSslContextFactoryInvalidConfigTest.java  |    8 +-
 .../security/DefaultSslContextFactoryTest.java     |   22 +-
 .../security/FileBasedSslContextFactoryTest.java   |  188 ++++
 .../PEMBasedSslContextFactoryConfigTest.java       |    9 +-
 ...tFactoryConfigWithMismatchingPasswordsTest.java |   97 ++
 ...ntextFactoryConfigWithUnencryptedKeysTest.java} |   14 +-
 ...PEMBasedSslContextFactoryInvalidConfigTest.java |    9 +-
 .../security/PEMBasedSslContextFactoryTest.java    |   17 +-
 .../PEMJKSSslContextFactoryConfigTest.java         |    9 +-
 .../AbstractFilesystemOwnershipCheckTest.java      |   13 +-
 .../apache/cassandra/service/ClientStateTest.java  |    8 +-
 .../cassandra/service/DiskFailurePolicyTest.java   |   81 +-
 .../cassandra/service/PartitionDenylistTest.java   |    3 +
 .../service/SSTablesGlobalTrackerTest.java         |   20 +-
 .../cassandra/service/StartupChecksTest.java       |   14 +-
 .../service/StorageServiceServerM3PTest.java       |    4 +-
 .../service/StorageServiceServerTest.java          |   17 +-
 .../cassandra/service/StorageServiceTest.java      |   60 +-
 ...ropertiesBasedFileSystemOwnershipCheckTest.java |    5 +-
 .../service/paxos/PaxosRepairHistoryTest.java      |    3 +-
 .../paxos/uncommitted/PaxosStateTrackerTest.java   |   36 +-
 .../service/reads/range/RangeCommandsTest.java     |    8 +-
 .../service/snapshot/SnapshotManifestTest.java     |    7 +-
 .../cassandra/streaming/StreamingTransferTest.java |    5 +-
 .../cassandra/tools/JMXCompatabilityTest.java      |  148 +--
 .../apache/cassandra/tools/OfflineToolUtils.java   |    3 +-
 .../tools/SSTableExportSchemaLoadingTest.java      |    5 +-
 .../cassandra/tools/SSTablePartitionsTest.java     |  654 +++++++++++++
 .../cassandra/tools/StandaloneSplitterTest.java    |    3 +-
 .../tools/StandaloneSplitterWithCQLTesterTest.java |    3 +-
 .../tools/StandaloneUpgraderOnSStablesTest.java    |    8 +-
 .../tools/StandaloneVerifierOnSSTablesTest.java    |   11 +-
 .../cassandra/tools/nodetool/ClientStatsTest.java  |    3 +-
 .../cassandra/tools/nodetool/ScrubToolTest.java    |    3 +-
 .../cassandra/tools/nodetool/TableStatsTest.java   |   29 +
 .../cassandra/tools/nodetool/TpStatsTest.java      |    5 +-
 .../nodetool/stats/TableStatsPrinterTest.java      |  200 +++-
 .../cassandra/transport/CQLUserAuditTest.java      |    3 +-
 .../apache/cassandra/transport/SerDeserTest.java   |   33 +
 .../apache/cassandra/utils/ByteBufferUtilTest.java |   33 +
 .../org/apache/cassandra/utils/Generators.java     |    4 +-
 .../cassandra/utils/btree/BTreeRemovalTest.java    |    3 +-
 .../apache/cassandra/utils/btree/BTreeTest.java    |    5 +-
 .../bytecomparable/AbstractTypeByteSourceTest.java |    8 +-
 .../bytecomparable/ByteSourceComparisonTest.java   |   32 +-
 .../bytecomparable/ByteSourceConversionTest.java   |    2 +-
 .../bytecomparable/ByteSourceInverseTest.java      |    4 +-
 .../bytecomparable/ByteSourceSequenceTest.java     |    2 +-
 .../bytecomparable/DecoratedKeyByteSourceTest.java |    2 +-
 tools/bin/sstablepartitions                        |   49 +
 .../io/sstable/StressCQLSSTableWriter.java         |   33 +-
 .../org/apache/cassandra/stress/StressGraph.java   |   61 +-
 .../cassandra/stress/report/StressMetrics.java     |   17 +-
 .../cassandra/stress/settings/CliOption.java       |   14 +-
 .../stress/settings/SettingsCredentials.java       |  129 +++
 .../cassandra/stress/settings/SettingsJMX.java     |   98 ++
 .../cassandra/stress/settings/SettingsMode.java    |   26 +-
 .../stress/settings/SettingsReporting.java         |  116 +++
 .../stress/settings/SettingsTransport.java         |   46 +-
 .../cassandra/stress/settings/StressSettings.java  |   29 +-
 .../cassandra/stress/util/JavaDriverClient.java    |   27 +-
 .../apache/cassandra/stress/util/JmxCollector.java |   12 +-
 .../stress/settings/SettingsCredentialsTest.java   |  126 +++
 767 files changed, 24255 insertions(+), 5408 deletions(-)
 rename doc/modules/cassandra/pages/architecture/{storage_engine.adoc => storage-engine.adoc} (100%)
 delete mode 100644 doc/modules/cassandra/pages/cql/dynamic_data_masking.adoc
 delete mode 100644 doc/modules/cassandra/pages/data_modeling/index.adoc
 rename doc/modules/cassandra/pages/{ => developing}/cql/SASI.adoc (100%)
 rename doc/modules/cassandra/pages/{ => developing}/cql/appendices.adoc (100%)
 rename doc/modules/cassandra/pages/{ => developing}/cql/changes.adoc (100%)
 rename doc/modules/cassandra/pages/{ => developing}/cql/cql_singlefile.adoc (100%)
 rename doc/modules/cassandra/pages/{ => developing}/cql/ddl.adoc (100%)
 rename doc/modules/cassandra/pages/{ => developing}/cql/definitions.adoc (100%)
 rename doc/modules/cassandra/pages/{ => developing}/cql/dml.adoc (100%)
 rename doc/modules/cassandra/pages/{ => developing}/cql/functions.adoc (100%)
 rename doc/modules/cassandra/pages/{ => developing}/cql/index.adoc (100%)
 rename doc/modules/cassandra/pages/{ => developing}/cql/indexes.adoc (100%)
 rename doc/modules/cassandra/pages/{ => developing}/cql/json.adoc (100%)
 rename doc/modules/cassandra/pages/{ => developing}/cql/mvs.adoc (100%)
 rename doc/modules/cassandra/pages/{ => developing}/cql/operators.adoc (100%)
 rename doc/modules/cassandra/pages/{ => developing}/cql/security.adoc (100%)
 rename doc/modules/cassandra/pages/{ => developing}/cql/triggers.adoc (100%)
 rename doc/modules/cassandra/pages/{ => developing}/cql/types.adoc (100%)
 rename doc/modules/cassandra/pages/{data_modeling/data_modeling_conceptual.adoc => developing/data-modeling/data-modeling_conceptual.adoc} (98%)
 rename doc/modules/cassandra/pages/{data_modeling/data_modeling_logical.adoc => developing/data-modeling/data-modeling_logical.adoc} (98%)
 rename doc/modules/cassandra/pages/{data_modeling/data_modeling_physical.adoc => developing/data-modeling/data-modeling_physical.adoc} (96%)
 rename doc/modules/cassandra/pages/{data_modeling/data_modeling_queries.adoc => developing/data-modeling/data-modeling_queries.adoc} (98%)
 rename doc/modules/cassandra/pages/{data_modeling/data_modeling_rdbms.adoc => developing/data-modeling/data-modeling_rdbms.adoc} (99%)
 rename doc/modules/cassandra/pages/{data_modeling/data_modeling_refining.adoc => developing/data-modeling/data-modeling_refining.adoc} (99%)
 rename doc/modules/cassandra/pages/{data_modeling/data_modeling_schema.adoc => developing/data-modeling/data-modeling_schema.adoc} (100%)
 rename doc/modules/cassandra/pages/{data_modeling/data_modeling_tools.adoc => developing/data-modeling/data-modeling_tools.adoc} (100%)
 rename doc/modules/cassandra/pages/{data_modeling => developing/data-modeling}/images/Figure_1_data_model.jpg (100%)
 rename doc/modules/cassandra/pages/{data_modeling => developing/data-modeling}/images/Figure_2_data_model.jpg (100%)
 rename doc/modules/cassandra/pages/{data_modeling => developing/data-modeling}/images/data_modeling_chebotko_logical.png (100%)
 rename doc/modules/cassandra/pages/{data_modeling => developing/data-modeling}/images/data_modeling_chebotko_physical.png (100%)
 rename doc/modules/cassandra/pages/{data_modeling => developing/data-modeling}/images/data_modeling_hotel_bucketing.png (100%)
 rename doc/modules/cassandra/pages/{data_modeling => developing/data-modeling}/images/data_modeling_hotel_erd.png (100%)
 rename doc/modules/cassandra/pages/{data_modeling => developing/data-modeling}/images/data_modeling_hotel_logical.png (100%)
 rename doc/modules/cassandra/pages/{data_modeling => developing/data-modeling}/images/data_modeling_hotel_physical.png (100%)
 rename doc/modules/cassandra/pages/{data_modeling => developing/data-modeling}/images/data_modeling_hotel_queries.png (100%)
 rename doc/modules/cassandra/pages/{data_modeling => developing/data-modeling}/images/data_modeling_hotel_relational.png (100%)
 rename doc/modules/cassandra/pages/{data_modeling => developing/data-modeling}/images/data_modeling_reservation_logical.png (100%)
 rename doc/modules/cassandra/pages/{data_modeling => developing/data-modeling}/images/data_modeling_reservation_physical.png (100%)
 create mode 100644 doc/modules/cassandra/pages/developing/data-modeling/index.adoc
 rename doc/modules/cassandra/pages/{data_modeling => developing/data-modeling}/intro.adoc (100%)
 rename doc/modules/cassandra/pages/{getting_started => getting-started}/configuring.adoc (100%)
 rename doc/modules/cassandra/pages/{getting_started => getting-started}/drivers.adoc (100%)
 create mode 100644 doc/modules/cassandra/pages/getting-started/index.adoc
 rename doc/modules/cassandra/pages/{getting_started => getting-started}/production.adoc (100%)
 rename doc/modules/cassandra/pages/{getting_started => getting-started}/querying.adoc (94%)
 rename doc/modules/cassandra/pages/{getting_started => getting-started}/quickstart.adoc (100%)
 delete mode 100644 doc/modules/cassandra/pages/getting_started/index.adoc
 rename doc/modules/cassandra/pages/{getting_started => installing}/installing.adoc (98%)
 rename doc/modules/cassandra/pages/{ => integrating}/plugins/index.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/configuration/cass_cl_archive_file.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/configuration/cass_env_sh_file.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/configuration/cass_jvm_options_file.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/configuration/cass_logback_xml_file.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/configuration/cass_rackdc_file.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/configuration/cass_topo_file.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/configuration/configuration.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/configuration/index.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/audit_logging.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/auditlogging.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/backups.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/bloom_filters.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/bulk_loading.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/cdc.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/compaction/index.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/compaction/lcs.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/compaction/stcs.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/compaction/twcs.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/compression.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/denylisting_partitions.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/fqllogging.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/hardware.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/hints.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/index.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/metrics.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/read_repair.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/repair.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/security.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/topo_changes.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/transientreplication.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/operating/virtualtables.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/tools/cassandra_stress.adoc (97%)
 rename doc/modules/cassandra/pages/{ => managing}/tools/cqlsh.adoc (97%)
 rename doc/modules/cassandra/pages/{ => managing}/tools/hash_password.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/tools/index.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/tools/sstable/index.adoc (93%)
 rename doc/modules/cassandra/pages/{ => managing}/tools/sstable/sstabledump.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/tools/sstable/sstableexpiredblockers.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/tools/sstable/sstablelevelreset.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/tools/sstable/sstableloader.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/tools/sstable/sstablemetadata.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/tools/sstable/sstableofflinerelevel.adoc (100%)
 create mode 100644 doc/modules/cassandra/pages/managing/tools/sstable/sstablepartitions.adoc
 rename doc/modules/cassandra/pages/{ => managing}/tools/sstable/sstablerepairedset.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/tools/sstable/sstablescrub.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/tools/sstable/sstablesplit.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/tools/sstable/sstableupgrade.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/tools/sstable/sstableutil.adoc (100%)
 rename doc/modules/cassandra/pages/{ => managing}/tools/sstable/sstableverify.adoc (100%)
 rename doc/modules/cassandra/pages/{new => managing}/virtualtables.adoc (100%)
 rename doc/modules/cassandra/pages/{ => overview}/faq/index.adoc (100%)
 create mode 100644 doc/modules/cassandra/pages/overview/terminology.adoc
 rename doc/modules/cassandra/pages/{getting_started => references}/java11.adoc (100%)
 copy test/conf/triggers/README.txt => doc/modules/cassandra/pages/tooling/cass-stress-deprecated (100%)
 copy test/conf/triggers/README.txt => doc/modules/cassandra/pages/tooling/generate-tokens (100%)
 copy test/conf/triggers/README.txt => doc/modules/cassandra/pages/tooling/hash-password-tool (100%)
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/SSTableScanner.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/SortedTableReaderLoadingBuilder.java
 delete mode 100644 src/java/org/apache/cassandra/io/sstable/format/VersionAndType.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormat.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormat.md
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormatPartitionWriter.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReader.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReaderLoadingBuilder.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableScanner.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableScrubber.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableVerifier.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableWriter.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndex.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndexBuilder.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIndexEarly.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/PartitionIterator.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexReader.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexReverseIterator.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/RowIndexWriter.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/SSTableIterator.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/SSTableReversedIterator.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/ScrubIterator.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/ScrubPartitionIterator.java
 create mode 100644 src/java/org/apache/cassandra/io/sstable/format/bti/TrieIndexEntry.java
 create mode 100644 src/java/org/apache/cassandra/io/tries/ReverseValueIterator.java
 create mode 100644 src/java/org/apache/cassandra/tools/SSTablePartitions.java
 create mode 100644 src/java/org/apache/cassandra/utils/JsonUtils.java
 create mode 100644 src/java/org/apache/cassandra/utils/RMIClientSocketFactoryImpl.java
 create mode 100644 src/resources/META-INF/services/org.apache.cassandra.io.sstable.format.SSTableFormat$Factory
 create mode 100644 test/conf/cassandra-pem-sslcontextfactory-mismatching-passwords.yaml
 create mode 100644 test/conf/cassandra-pem-sslcontextfactory-unencryptedkeys.yaml
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-CompressionInfo.db
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Data.db
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Digest.crc32
 copy test/data/legacy-sstables/{na/legacy_tables/legacy_na_clust/na-1-big-Filter.db => da/legacy_tables/legacy_da_clust/da-1-bti-Filter.db} (100%)
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Partitions.db
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Rows.db
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-Statistics.db
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_clust/da-1-bti-TOC.txt
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-CompressionInfo.db
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Data.db
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Digest.crc32
 copy test/data/legacy-sstables/{na/legacy_tables/legacy_na_clust/na-1-big-Filter.db => da/legacy_tables/legacy_da_clust_counter/da-1-bti-Filter.db} (100%)
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Partitions.db
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Rows.db
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-Statistics.db
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_clust_counter/da-1-bti-TOC.txt
 copy test/data/legacy-sstables/{nb/legacy_tables/legacy_nb_simple/nb-1-big-CompressionInfo.db => da/legacy_tables/legacy_da_simple/da-1-bti-CompressionInfo.db} (100%)
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Data.db
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Digest.crc32
 copy test/data/legacy-sstables/{na/legacy_tables/legacy_na_clust/na-1-big-Filter.db => da/legacy_tables/legacy_da_simple/da-1-bti-Filter.db} (100%)
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Partitions.db
 copy test/{conf/triggers/README.txt => data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Rows.db} (100%)
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-Statistics.db
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_simple/da-1-bti-TOC.txt
 copy test/data/legacy-sstables/{nb/legacy_tables/legacy_nb_simple_counter/nb-1-big-CompressionInfo.db => da/legacy_tables/legacy_da_simple_counter/da-1-bti-CompressionInfo.db} (100%)
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Data.db
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Digest.crc32
 copy test/data/legacy-sstables/{na/legacy_tables/legacy_na_clust/na-1-big-Filter.db => da/legacy_tables/legacy_da_simple_counter/da-1-bti-Filter.db} (100%)
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Partitions.db
 copy test/{conf/triggers/README.txt => data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Rows.db} (100%)
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-Statistics.db
 create mode 100644 test/data/legacy-sstables/da/legacy_tables/legacy_da_simple_counter/da-1-bti-TOC.txt
 create mode 100644 test/distributed/org/apache/cassandra/distributed/impl/CollectingRMIServerSocketFactoryImpl.java
 create mode 100644 test/distributed/org/apache/cassandra/distributed/impl/IsolatedJmx.java
 create mode 100644 test/distributed/org/apache/cassandra/distributed/test/CompactionOverlappingSSTableTest.java
 create mode 100644 test/distributed/org/apache/cassandra/distributed/test/IndexDroppingTest.java
 create mode 100644 test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailPartitionSizeTest.java
 create mode 100644 test/distributed/org/apache/cassandra/distributed/test/guardrails/GuardrailPartitionTombstonesTest.java
 create mode 100644 test/distributed/org/apache/cassandra/distributed/test/jmx/JMXFeatureTest.java
 copy test/{unit/org/apache/cassandra/service/SystemPropertiesBasedFileSystemOwnershipCheckTest.java => distributed/org/apache/cassandra/distributed/test/topology/DecommissionAvoidReadTimeoutsTest.java} (62%)
 create mode 100644 test/distributed/org/apache/cassandra/distributed/test/topology/DecommissionAvoidTimeouts.java
 copy test/{unit/org/apache/cassandra/service/SystemPropertiesBasedFileSystemOwnershipCheckTest.java => distributed/org/apache/cassandra/distributed/test/topology/DecommissionAvoidWriteTimeoutsTest.java} (62%)
 create mode 100644 test/distributed/org/apache/cassandra/distributed/util/Coordinators.java
 create mode 100644 test/distributed/org/apache/cassandra/distributed/util/TwoWay.java
 create mode 100644 test/distributed/org/apache/cassandra/distributed/util/byterewrite/StatusChangeListener.java
 create mode 100644 test/distributed/org/apache/cassandra/distributed/util/byterewrite/Undead.java
 create mode 100644 test/unit/org/apache/cassandra/cql3/DecoratedKeyPrefixesTest.java
 create mode 100644 test/unit/org/apache/cassandra/db/CorruptPrimaryIndexTest.java
 create mode 100644 test/unit/org/apache/cassandra/db/compaction/PartialCompactionsTest.java
 create mode 100644 test/unit/org/apache/cassandra/db/guardrails/GuardrailMaximumTimestampTest.java
 create mode 100644 test/unit/org/apache/cassandra/db/guardrails/GuardrailMinimumTimestampTest.java
 create mode 100644 test/unit/org/apache/cassandra/db/guardrails/GuardrailPartitionSizeTest.java
 create mode 100644 test/unit/org/apache/cassandra/db/guardrails/GuardrailPartitionTombstonesTest.java
 create mode 100644 test/unit/org/apache/cassandra/db/tries/TrieToMermaid.java
 copy test/unit/org/apache/cassandra/{security/CustomSslContextFactoryInvalidConfigTest.java => db/tries/TrieToMermaidTest.java} (50%)
 create mode 100644 test/unit/org/apache/cassandra/db/virtual/ClientsTableKeyspaceColTest.java
 create mode 100644 test/unit/org/apache/cassandra/io/filesystem/ForwardingFileChannel.java
 create mode 100644 test/unit/org/apache/cassandra/io/filesystem/ForwardingFileSystem.java
 create mode 100644 test/unit/org/apache/cassandra/io/filesystem/ForwardingFileSystemProvider.java
 create mode 100644 test/unit/org/apache/cassandra/io/filesystem/ForwardingPath.java
 create mode 100644 test/unit/org/apache/cassandra/io/filesystem/ListenableFileSystem.java
 create mode 100644 test/unit/org/apache/cassandra/io/sstable/SSTableFormatTest.java
 delete mode 100644 test/unit/org/apache/cassandra/io/sstable/SSTableFormatTypeTest.java
 create mode 100644 test/unit/org/apache/cassandra/io/sstable/format/AbstractTestVersionSupportedFeatures.java
 delete mode 100644 test/unit/org/apache/cassandra/io/sstable/format/VersionAndTypeTest.java
 create mode 100644 test/unit/org/apache/cassandra/io/sstable/format/big/VersionSupportedFeaturesTest.java
 create mode 100644 test/unit/org/apache/cassandra/io/sstable/format/bti/PartitionIndexTest.java
 create mode 100644 test/unit/org/apache/cassandra/io/sstable/format/bti/RowIndexTest.java
 create mode 100644 test/unit/org/apache/cassandra/io/sstable/format/bti/VersionSupportedFeaturesTest.java
 create mode 100644 test/unit/org/apache/cassandra/io/util/FileSystems.java
 create mode 100644 test/unit/org/apache/cassandra/security/FileBasedSslContextFactoryTest.java
 create mode 100644 test/unit/org/apache/cassandra/security/PEMBasedSslContextFactoryConfigWithMismatchingPasswordsTest.java
 copy test/unit/org/apache/cassandra/security/{PEMBasedSslContextFactoryConfigTest.java => PEMBasedSslContextFactoryConfigWithUnencryptedKeysTest.java} (83%)
 create mode 100644 test/unit/org/apache/cassandra/tools/SSTablePartitionsTest.java
 create mode 100755 tools/bin/sstablepartitions
 create mode 100644 tools/stress/src/org/apache/cassandra/stress/settings/SettingsCredentials.java
 create mode 100644 tools/stress/src/org/apache/cassandra/stress/settings/SettingsJMX.java
 create mode 100644 tools/stress/src/org/apache/cassandra/stress/settings/SettingsReporting.java
 create mode 100644 tools/stress/test/unit/org/apache/cassandra/stress/settings/SettingsCredentialsTest.java


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


[cassandra] 04/06: Allow CQL queries on multiple indexes without ALLOW FILTERING

Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

maedhroz pushed a commit to branch cep-7-sai
in repository https://gitbox.apache.org/repos/asf/cassandra.git

commit b661711765859181d6452cb5c5b588b06259c142
Author: Andrés de la Peña <a....@gmail.com>
AuthorDate: Mon Apr 24 16:39:26 2023 +0100

    Allow CQL queries on multiple indexes without ALLOW FILTERING
    
    patch by Andrés de la Peña; reviewed by Berenguer Blasi for CASSANDRA-18217
---
 .../restrictions/ClusteringColumnRestrictions.java |  25 +-
 .../cql3/restrictions/CustomIndexExpression.java   |  18 ++
 .../cql3/restrictions/IndexRestrictions.java       |  45 +++
 .../cql3/restrictions/MultiColumnRestriction.java  |  70 +++--
 .../PartitionKeySingleRestrictionSet.java          |   6 +-
 .../cassandra/cql3/restrictions/Restriction.java   |  13 +-
 .../cql3/restrictions/RestrictionSet.java          |  41 ++-
 .../cql3/restrictions/RestrictionSetWrapper.java   |  11 +-
 .../cql3/restrictions/SingleColumnRestriction.java |  45 ++-
 .../cql3/restrictions/StatementRestrictions.java   |  23 +-
 .../cassandra/cql3/restrictions/TokenFilter.java   |  11 +-
 .../cql3/restrictions/TokenRestriction.java        |   9 +-
 .../cassandra/cql3/statements/SelectStatement.java |  11 +-
 src/java/org/apache/cassandra/index/Index.java     |  21 +-
 .../cassandra/index/sai/StorageAttachedIndex.java  |   6 +
 .../ClusteringColumnRestrictionsTest.java          | 301 +++++++-------------
 .../apache/cassandra/index/CustomIndexTest.java    |  26 +-
 .../index/sai/cql/AllowFilteringTest.java          | 314 +++++++++++++++++++++
 .../cassandra/index/sai/cql/BaseDataModel.java     |  13 +-
 .../index/sai/cql/CollectionIndexingTest.java      |  39 ++-
 .../sai/cql/CompositePartitionKeyIndexTest.java    |   4 +-
 .../cassandra/index/sai/cql/IndexQuerySupport.java |  79 +++---
 .../sai/cql/MixedIndexImplementationsTest.java     |  32 +--
 .../index/sai/cql/MultipleColumnIndexTest.java     |  10 +-
 .../cassandra/index/sai/cql/types/QuerySet.java    |  16 +-
 .../index/sai/disk/SelectiveIntersectionTest.java  |  12 +-
 .../index/sai/disk/SingleNodeQueryFailureTest.java |   6 +-
 27 files changed, 802 insertions(+), 405 deletions(-)

diff --git a/src/java/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictions.java
index 146ec20557..ddd9601c83 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictions.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictions.java
@@ -19,6 +19,8 @@ package org.apache.cassandra.cql3.restrictions;
 
 import java.util.*;
 
+import javax.annotation.Nullable;
+
 import org.apache.cassandra.db.guardrails.Guardrails;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.TableMetadata;
@@ -42,18 +44,13 @@ final class ClusteringColumnRestrictions extends RestrictionSetWrapper
     /**
      * The composite type.
      */
-    protected final ClusteringComparator comparator;
+    private final ClusteringComparator comparator;
 
     /**
      * <code>true</code> if filtering is allowed for this restriction, <code>false</code> otherwise
      */
     private final boolean allowFiltering;
 
-    public ClusteringColumnRestrictions(TableMetadata table)
-    {
-        this(table, false);
-    }
-
     public ClusteringColumnRestrictions(TableMetadata table, boolean allowFiltering)
     {
         this(table.comparator, new RestrictionSet(), allowFiltering);
@@ -68,14 +65,16 @@ final class ClusteringColumnRestrictions extends RestrictionSetWrapper
         this.allowFiltering = allowFiltering;
     }
 
-    public ClusteringColumnRestrictions mergeWith(Restriction restriction) throws InvalidRequestException
+    public ClusteringColumnRestrictions mergeWith(Restriction restriction, @Nullable IndexRegistry indexRegistry) throws InvalidRequestException
     {
         SingleRestriction newRestriction = (SingleRestriction) restriction;
         RestrictionSet newRestrictionSet = restrictions.addRestriction(newRestriction);
 
-        if (!isEmpty() && !allowFiltering)
+        if (!isEmpty() && !allowFiltering && (indexRegistry == null || !newRestriction.hasSupportingIndex(indexRegistry)))
         {
             SingleRestriction lastRestriction = restrictions.lastRestriction();
+            assert lastRestriction != null;
+
             ColumnMetadata lastRestrictionStart = lastRestriction.getFirstColumn();
             ColumnMetadata newRestrictionStart = restriction.getFirstColumn();
 
@@ -156,7 +155,7 @@ final class ClusteringColumnRestrictions extends RestrictionSetWrapper
      * @return <code>true</code> if any of the underlying restriction is a CONTAINS or CONTAINS KEY,
      * <code>false</code> otherwise
      */
-    public final boolean hasContains()
+    public boolean hasContains()
     {
         for (SingleRestriction restriction : restrictions)
         {
@@ -172,7 +171,7 @@ final class ClusteringColumnRestrictions extends RestrictionSetWrapper
      * @return <code>true</code> if any of the underlying restriction is a slice restrictions,
      * <code>false</code> otherwise
      */
-    public final boolean hasSlice()
+    public boolean hasSlice()
     {
         for (SingleRestriction restriction : restrictions)
         {
@@ -188,7 +187,7 @@ final class ClusteringColumnRestrictions extends RestrictionSetWrapper
      * @return <code>true</code> if any underlying restrictions require filtering, <code>false</code>
      * otherwise
      */
-    public final boolean needFiltering()
+    public boolean needFiltering()
     {
         int position = 0;
 
@@ -204,7 +203,7 @@ final class ClusteringColumnRestrictions extends RestrictionSetWrapper
     }
 
     @Override
-    public void addRowFilterTo(RowFilter filter,
+    public void addToRowFilter(RowFilter filter,
                                IndexRegistry indexRegistry,
                                QueryOptions options) throws InvalidRequestException
     {
@@ -215,7 +214,7 @@ final class ClusteringColumnRestrictions extends RestrictionSetWrapper
             // We ignore all the clustering columns that can be handled by slices.
             if (handleInFilter(restriction, position) || restriction.hasSupportingIndex(indexRegistry))
             {
-                restriction.addRowFilterTo(filter, indexRegistry, options);
+                restriction.addToRowFilter(filter, indexRegistry, options);
                 continue;
             }
 
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/CustomIndexExpression.java b/src/java/org/apache/cassandra/cql3/restrictions/CustomIndexExpression.java
index 7a5fff62b4..569418b3f4 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/CustomIndexExpression.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/CustomIndexExpression.java
@@ -22,6 +22,7 @@ import java.util.Objects;
 import org.apache.cassandra.cql3.*;
 import org.apache.cassandra.db.filter.RowFilter;
 import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.schema.TableMetadata;
 
 public class CustomIndexExpression
@@ -55,6 +56,23 @@ public class CustomIndexExpression
                                         value.bindAndGet(options));
     }
 
+    /**
+     * Returns whether this expression would need filtering if the specified index group were used.
+     *
+     * @param indexGroup an index group
+     * @return {@code true} if this would need filtering if {@code indexGroup} were used, {@code false} otherwise
+     */
+    public boolean needsFiltering(Index.Group indexGroup)
+    {
+        String indexName = targetIndex.getName();
+        for (Index index : indexGroup.getIndexes())
+        {
+            if (index.getIndexMetadata().name.equals(indexName))
+                return false;
+        }
+        return true;
+    }
+
     public String toCQLString()
     {
         return String.format("expr(%s,%s)", targetIndex.toCQLString(), valueRaw.getText());
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/IndexRestrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/IndexRestrictions.java
index fd89d1b470..fe01e41887 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/IndexRestrictions.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/IndexRestrictions.java
@@ -22,6 +22,8 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.cassandra.cql3.QualifiedName;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.IndexRegistry;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.commons.lang3.builder.ToStringBuilder;
@@ -63,6 +65,49 @@ public class IndexRestrictions
         return customExpressions;
     }
 
+    /**
+     * Returns whether these restrictions would need filtering if the specified index registry were used.
+     *
+     * @param indexRegistry an index registry
+     * @return {@code true} if this would need filtering if {@code indexRegistry} were used, {@code false} otherwise
+     */
+    public boolean needsFiltering(IndexRegistry indexRegistry)
+    {
+        if (isEmpty())
+            return false;
+
+        for (Index.Group group : indexRegistry.listIndexGroups())
+        {
+            if (!needsFiltering(group))
+                return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * Returns whether these restrictions would need filtering if the specified index group were used.
+     *
+     * @param indexGroup an index group
+     * @return {@code true} if this would need filtering if {@code indexGroup} were used, {@code false} otherwise
+     */
+    private boolean needsFiltering(Index.Group indexGroup)
+    {
+        for (Restrictions restrictions : regularRestrictions)
+        {
+            if (restrictions.needsFiltering(indexGroup))
+                return true;
+        }
+
+        for (CustomIndexExpression restriction : customExpressions)
+        {
+            if (restriction.needsFiltering(indexGroup))
+                return true;
+        }
+
+        return false;
+    }
+
     static InvalidRequestException invalidIndex(QualifiedName indexName, TableMetadata table)
     {
         return new InvalidRequestException(String.format(INVALID_INDEX, indexName.getName(), table));
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/MultiColumnRestriction.java b/src/java/org/apache/cassandra/cql3/restrictions/MultiColumnRestriction.java
index 22c000e3b5..37c64c614d 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/MultiColumnRestriction.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/MultiColumnRestriction.java
@@ -128,6 +128,28 @@ public abstract class MultiColumnRestriction implements SingleRestriction
         for (Index index : indexRegistry.listIndexes())
            if (isSupportedBy(index))
                return true;
+
+        return false;
+    }
+
+    @Override
+    public boolean needsFiltering(Index.Group indexGroup)
+    {
+        for (ColumnMetadata column : columnDefs)
+        {
+            if (!isSupportedBy(indexGroup, column))
+                return true;
+        }
+        return false;
+    }
+
+    private boolean isSupportedBy(Index.Group indexGroup, ColumnMetadata column)
+    {
+        for (Index index : indexGroup.getIndexes())
+        {
+            if (isSupportedBy(index, column))
+                return true;
+        }
         return false;
     }
 
@@ -138,7 +160,17 @@ public abstract class MultiColumnRestriction implements SingleRestriction
      * @return <code>true</code> this type of restriction is supported by the specified index,
      * <code>false</code> otherwise.
      */
-    protected abstract boolean isSupportedBy(Index index);
+    private boolean isSupportedBy(Index index)
+    {
+        for (ColumnMetadata column : columnDefs)
+        {
+            if (isSupportedBy(index, column))
+                return true;
+        }
+        return false;
+    }
+
+    protected abstract boolean isSupportedBy(Index index, ColumnMetadata def);
 
     public static class EQRestriction extends MultiColumnRestriction
     {
@@ -176,12 +208,9 @@ public abstract class MultiColumnRestriction implements SingleRestriction
         }
 
         @Override
-        protected boolean isSupportedBy(Index index)
+        protected boolean isSupportedBy(Index index, ColumnMetadata column)
         {
-            for(ColumnMetadata column : columnDefs)
-                if (index.supportsExpression(column, Operator.EQ))
-                    return true;
-            return false;
+            return index.supportsExpression(column, Operator.EQ);
         }
 
         @Override
@@ -198,7 +227,7 @@ public abstract class MultiColumnRestriction implements SingleRestriction
         }
 
         @Override
-        public final void addRowFilterTo(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
+        public final void addToRowFilter(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
         {
             Tuples.Value t = ((Tuples.Value) value.bind(options));
             List<ByteBuffer> values = t.getElements();
@@ -246,16 +275,13 @@ public abstract class MultiColumnRestriction implements SingleRestriction
         }
 
         @Override
-        protected boolean isSupportedBy(Index index)
+        protected boolean isSupportedBy(Index index, ColumnMetadata column)
         {
-            for (ColumnMetadata column: columnDefs)
-                if (index.supportsExpression(column, Operator.IN))
-                    return true;
-            return false;
+            return index.supportsExpression(column, Operator.IN);
         }
 
         @Override
-        public final void addRowFilterTo(RowFilter filter,
+        public final void addToRowFilter(RowFilter filter,
                                          IndexRegistry indexRegistry,
                                          QueryOptions options)
         {
@@ -443,12 +469,9 @@ public abstract class MultiColumnRestriction implements SingleRestriction
         }
 
         @Override
-        protected boolean isSupportedBy(Index index)
+        protected boolean isSupportedBy(Index index, ColumnMetadata column)
         {
-            for(ColumnMetadata def : columnDefs)
-                if (slice.isSupportedBy(def, index))
-                    return true;
-            return false;
+            return slice.isSupportedBy(column, index);
         }
 
         @Override
@@ -499,7 +522,7 @@ public abstract class MultiColumnRestriction implements SingleRestriction
         }
 
         @Override
-        public final void addRowFilterTo(RowFilter filter,
+        public final void addToRowFilter(RowFilter filter,
                                          IndexRegistry indexRegistry,
                                          QueryOptions options)
         {
@@ -573,12 +596,9 @@ public abstract class MultiColumnRestriction implements SingleRestriction
         }
 
         @Override
-        protected boolean isSupportedBy(Index index)
+        protected boolean isSupportedBy(Index index, ColumnMetadata column)
         {
-            for(ColumnMetadata column : columnDefs)
-                if (index.supportsExpression(column, Operator.IS_NOT))
-                    return true;
-            return false;
+            return index.supportsExpression(column, Operator.IS_NOT);
         }
 
         @Override
@@ -588,7 +608,7 @@ public abstract class MultiColumnRestriction implements SingleRestriction
         }
 
         @Override
-        public final void addRowFilterTo(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
+        public final void addToRowFilter(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
         {
             throw new UnsupportedOperationException("Secondary indexes do not support IS NOT NULL restrictions");
         }
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeySingleRestrictionSet.java b/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeySingleRestrictionSet.java
index a6f227a91d..b26f07f39c 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeySingleRestrictionSet.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/PartitionKeySingleRestrictionSet.java
@@ -42,7 +42,7 @@ final class PartitionKeySingleRestrictionSet extends RestrictionSetWrapper imple
     /**
      * The composite type.
      */
-    protected final ClusteringComparator comparator;
+    private final ClusteringComparator comparator;
 
     public PartitionKeySingleRestrictionSet(ClusteringComparator comparator)
     {
@@ -126,13 +126,13 @@ final class PartitionKeySingleRestrictionSet extends RestrictionSetWrapper imple
     }
 
     @Override
-    public void addRowFilterTo(RowFilter filter,
+    public void addToRowFilter(RowFilter filter,
                                IndexRegistry indexRegistry,
                                QueryOptions options)
     {
         for (SingleRestriction restriction : restrictions)
         {
-             restriction.addRowFilterTo(filter, indexRegistry, options);
+             restriction.addToRowFilter(filter, indexRegistry, options);
         }
     }
 
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/Restriction.java b/src/java/org/apache/cassandra/cql3/restrictions/Restriction.java
index 91dedad953..e2ab1aba95 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/Restriction.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/Restriction.java
@@ -19,6 +19,7 @@ package org.apache.cassandra.cql3.restrictions;
 
 import java.util.List;
 
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.functions.Function;
@@ -66,7 +67,15 @@ public interface Restriction
      * @param indexRegistry the index registry
      * @return <code>true</code> if the restriction is on indexed columns, <code>false</code>
      */
-    public boolean hasSupportingIndex(IndexRegistry indexRegistry);
+    boolean hasSupportingIndex(IndexRegistry indexRegistry);
+
+    /**
+     * Returns whether this restriction would need filtering if the specified index group were used.
+     *
+     * @param indexGroup an index group
+     * @return {@code true} if this would need filtering if {@code indexGroup} were used, {@code false} otherwise
+     */
+    boolean needsFiltering(Index.Group indexGroup);
 
     /**
      * Adds to the specified row filter the expressions corresponding to this <code>Restriction</code>.
@@ -75,7 +84,7 @@ public interface Restriction
      * @param indexRegistry the index registry
      * @param options the query options
      */
-    public void addRowFilterTo(RowFilter filter,
+    public void addToRowFilter(RowFilter filter,
                                IndexRegistry indexRegistry,
                                QueryOptions options);
 }
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSet.java b/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSet.java
index 344498ce0e..86aa269dcd 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSet.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSet.java
@@ -21,10 +21,10 @@ import java.util.*;
 
 import com.google.common.collect.AbstractIterator;
 
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.functions.Function;
-import org.apache.cassandra.cql3.restrictions.SingleColumnRestriction.ContainsRestriction;
 import org.apache.cassandra.db.filter.RowFilter;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.index.IndexRegistry;
@@ -56,7 +56,7 @@ final class RestrictionSet implements Restrictions, Iterable<SingleRestriction>
     /**
      * The restrictions per column.
      */
-    protected final TreeMap<ColumnMetadata, SingleRestriction> restrictions;
+    private final TreeMap<ColumnMetadata, SingleRestriction> restrictions;
 
     /**
      * {@code true} if it contains multi-column restrictions, {@code false} otherwise.
@@ -93,10 +93,21 @@ final class RestrictionSet implements Restrictions, Iterable<SingleRestriction>
     }
 
     @Override
-    public void addRowFilterTo(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options) throws InvalidRequestException
+    public void addToRowFilter(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options) throws InvalidRequestException
     {
         for (Restriction restriction : restrictions.values())
-            restriction.addRowFilterTo(filter, indexRegistry, options);
+            restriction.addToRowFilter(filter, indexRegistry, options);
+    }
+
+    @Override
+    public boolean needsFiltering(Index.Group indexGroup)
+    {
+        for (SingleRestriction restriction : restrictions.values())
+        {
+            if (restriction.needsFiltering(indexGroup))
+                return true;
+        }
+        return false;
     }
 
     @Override
@@ -223,7 +234,7 @@ final class RestrictionSet implements Restrictions, Iterable<SingleRestriction>
     }
 
     @Override
-    public final boolean hasSupportingIndex(IndexRegistry indexRegistry)
+    public boolean hasSupportingIndex(IndexRegistry indexRegistry)
     {
         for (Restriction restriction : restrictions.values())
         {
@@ -281,26 +292,6 @@ final class RestrictionSet implements Restrictions, Iterable<SingleRestriction>
                                    : restriction.mergeWith(otherRestriction);
     }
 
-    /**
-     * Checks if the restrictions contains multiple contains, contains key, or map[key] = value.
-     *
-     * @return <code>true</code> if the restrictions contains multiple contains, contains key, or ,
-     * map[key] = value; <code>false</code> otherwise
-     */
-    public final boolean hasMultipleContains()
-    {
-        int numberOfContains = 0;
-        for (SingleRestriction restriction : restrictions.values())
-        {
-            if (restriction.isContains())
-            {
-                ContainsRestriction contains = (ContainsRestriction) restriction;
-                numberOfContains += (contains.numberOfValues() + contains.numberOfKeys() + contains.numberOfEntries());
-            }
-        }
-        return numberOfContains > 1;
-    }
-
     @Override
     public Iterator<SingleRestriction> iterator()
     {
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSetWrapper.java b/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSetWrapper.java
index 8c89bca314..ac092740c0 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSetWrapper.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/RestrictionSetWrapper.java
@@ -24,6 +24,7 @@ import java.util.Set;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.commons.lang3.builder.ToStringStyle;
 
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.cql3.QueryOptions;
 import org.apache.cassandra.cql3.functions.Function;
@@ -46,11 +47,11 @@ class RestrictionSetWrapper implements Restrictions
         this.restrictions = restrictions;
     }
 
-    public void addRowFilterTo(RowFilter filter,
+    public void addToRowFilter(RowFilter filter,
                                IndexRegistry indexRegistry,
                                QueryOptions options)
     {
-        restrictions.addRowFilterTo(filter, indexRegistry, options);
+        restrictions.addToRowFilter(filter, indexRegistry, options);
     }
 
     public List<ColumnMetadata> getColumnDefs()
@@ -89,6 +90,12 @@ class RestrictionSetWrapper implements Restrictions
         return restrictions.hasSupportingIndex(indexRegistry);
     }
 
+    @Override
+    public boolean needsFiltering(Index.Group indexGroup)
+    {
+        return restrictions.needsFiltering(indexGroup);
+    }
+
     public ColumnMetadata getFirstColumn()
     {
         return restrictions.getFirstColumn();
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/SingleColumnRestriction.java b/src/java/org/apache/cassandra/cql3/restrictions/SingleColumnRestriction.java
index 5ce78ae577..a398762d2f 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/SingleColumnRestriction.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/SingleColumnRestriction.java
@@ -81,6 +81,16 @@ public abstract class SingleColumnRestriction implements SingleRestriction
         return false;
     }
 
+    @Override
+    public boolean needsFiltering(Index.Group indexGroup)
+    {
+        for (Index index : indexGroup.getIndexes())
+            if (isSupportedBy(index))
+                return false;
+
+        return true;
+    }
+
     @Override
     public final SingleRestriction mergeWith(SingleRestriction otherRestriction)
     {
@@ -151,7 +161,7 @@ public abstract class SingleColumnRestriction implements SingleRestriction
         }
 
         @Override
-        public void addRowFilterTo(RowFilter filter,
+        public void addToRowFilter(RowFilter filter,
                                    IndexRegistry indexRegistry,
                                    QueryOptions options)
         {
@@ -215,7 +225,7 @@ public abstract class SingleColumnRestriction implements SingleRestriction
         }
 
         @Override
-        public void addRowFilterTo(RowFilter filter,
+        public void addToRowFilter(RowFilter filter,
                                    IndexRegistry indexRegistry,
                                    QueryOptions options)
         {
@@ -388,7 +398,7 @@ public abstract class SingleColumnRestriction implements SingleRestriction
         }
 
         @Override
-        public void addRowFilterTo(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
+        public void addToRowFilter(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
         {
             for (Bound b : Bound.values())
                 if (hasBound(b))
@@ -417,10 +427,10 @@ public abstract class SingleColumnRestriction implements SingleRestriction
     // This holds CONTAINS, CONTAINS_KEY, and map[key] = value restrictions because we might want to have any combination of them.
     public static final class ContainsRestriction extends SingleColumnRestriction
     {
-        private List<Term> values = new ArrayList<>(); // for CONTAINS
-        private List<Term> keys = new ArrayList<>(); // for CONTAINS_KEY
-        private List<Term> entryKeys = new ArrayList<>(); // for map[key] = value
-        private List<Term> entryValues = new ArrayList<>(); // for map[key] = value
+        private final List<Term> values = new ArrayList<>(); // for CONTAINS
+        private final List<Term> keys = new ArrayList<>(); // for CONTAINS_KEY
+        private final List<Term> entryKeys = new ArrayList<>(); // for map[key] = value
+        private final List<Term> entryValues = new ArrayList<>(); // for map[key] = value
 
         public ContainsRestriction(ColumnMetadata columnDef, Term t, boolean isKey)
         {
@@ -478,7 +488,7 @@ public abstract class SingleColumnRestriction implements SingleRestriction
         }
 
         @Override
-        public void addRowFilterTo(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
+        public void addToRowFilter(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
         {
             for (ByteBuffer value : bindAndGet(values, options))
                 filter.add(columnDef, Operator.CONTAINS, value);
@@ -509,6 +519,21 @@ public abstract class SingleColumnRestriction implements SingleRestriction
             return supported;
         }
 
+        @Override
+        public boolean needsFiltering(Index.Group indexGroup)
+        {
+            // multiple contains might require filtering on some indexes, since that is equivalent to a disjunction (or)
+            boolean hasMultipleContains = (numberOfValues() + numberOfKeys() + numberOfEntries()) > 1;
+
+            for (Index index : indexGroup.getIndexes())
+            {
+                if (isSupportedBy(index) && !(hasMultipleContains && index.filtersMultipleContains()))
+                    return false;
+            }
+
+            return true;
+        }
+
         public int numberOfValues()
         {
             return values.size();
@@ -617,7 +642,7 @@ public abstract class SingleColumnRestriction implements SingleRestriction
         }
 
         @Override
-        public void addRowFilterTo(RowFilter filter,
+        public void addToRowFilter(RowFilter filter,
                                    IndexRegistry indexRegistry,
                                    QueryOptions options)
         {
@@ -693,7 +718,7 @@ public abstract class SingleColumnRestriction implements SingleRestriction
         }
 
         @Override
-        public void addRowFilterTo(RowFilter filter,
+        public void addToRowFilter(RowFilter filter,
                                    IndexRegistry indexRegistry,
                                    QueryOptions options)
         {
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
index 52deaabbfa..bcf1642aac 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
@@ -196,13 +196,13 @@ public final class StatementRestrictions
                 if (!type.allowUseOfSecondaryIndices() || !restriction.hasSupportingIndex(indexRegistry))
                     throw new InvalidRequestException(String.format("LIKE restriction is only supported on properly " +
                                                                     "indexed columns. %s is not valid.",
-                                                                    relation.toString()));
+                                                                    relation));
 
-                addRestriction(restriction);
+                addRestriction(restriction, indexRegistry);
             }
             else
             {
-                addRestriction(relation.toRestriction(table, boundNames));
+                addRestriction(relation.toRestriction(table, boundNames), indexRegistry);
             }
         }
 
@@ -296,13 +296,13 @@ public final class StatementRestrictions
         return !tableNullable.allowFilteringImplicitly();
     }
 
-    private void addRestriction(Restriction restriction)
+    private void addRestriction(Restriction restriction, IndexRegistry indexRegistry)
     {
         ColumnMetadata def = restriction.getFirstColumn();
         if (def.isPartitionKey())
             partitionKeyRestrictions = partitionKeyRestrictions.mergeWith(restriction);
         else if (def.isClusteringColumn())
-            clusteringColumnsRestrictions = clusteringColumnsRestrictions.mergeWith(restriction);
+            clusteringColumnsRestrictions = clusteringColumnsRestrictions.mergeWith(restriction, indexRegistry);
         else
             nonPrimaryKeyRestrictions = nonPrimaryKeyRestrictions.addRestriction((SingleRestriction) restriction);
     }
@@ -682,7 +682,7 @@ public final class StatementRestrictions
 
         RowFilter filter = RowFilter.create();
         for (Restrictions restrictions : filterRestrictions.getRestrictions())
-            restrictions.addRowFilterTo(filter, indexRegistry, options);
+            restrictions.addToRowFilter(filter, indexRegistry, options);
 
         for (CustomIndexExpression expression : filterRestrictions.getCustomIndexExpressions())
             expression.addToRowFilter(filter, table, options);
@@ -868,16 +868,17 @@ public final class StatementRestrictions
      * Checks if the query need to use filtering.
      * @return <code>true</code> if the query need to use filtering, <code>false</code> otherwise.
      */
-    public boolean needFiltering()
+    public boolean needFiltering(TableMetadata table)
     {
+        IndexRegistry indexRegistry = IndexRegistry.obtain(table);
+        if (filterRestrictions.needsFiltering(indexRegistry))
+            return true;
+
         int numberOfRestrictions = filterRestrictions.getCustomIndexExpressions().size();
         for (Restrictions restrictions : filterRestrictions.getRestrictions())
             numberOfRestrictions += restrictions.size();
 
-        return numberOfRestrictions > 1
-                || (numberOfRestrictions == 0 && !clusteringColumnsRestrictions.isEmpty())
-                || (numberOfRestrictions != 0
-                        && nonPrimaryKeyRestrictions.hasMultipleContains());
+        return numberOfRestrictions == 0 && !clusteringColumnsRestrictions.isEmpty();
     }
 
     private void validateSecondaryIndexSelections()
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/TokenFilter.java b/src/java/org/apache/cassandra/cql3/restrictions/TokenFilter.java
index 9f67cc0547..42e3acf716 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/TokenFilter.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/TokenFilter.java
@@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableRangeSet;
 import com.google.common.collect.Range;
 import com.google.common.collect.RangeSet;
 
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.cql3.QueryOptions;
@@ -280,9 +281,15 @@ final class TokenFilter implements PartitionKeyRestrictions
     }
 
     @Override
-    public void addRowFilterTo(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
+    public void addToRowFilter(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
     {
-        restrictions.addRowFilterTo(filter, indexRegistry, options);
+        restrictions.addToRowFilter(filter, indexRegistry, options);
+    }
+
+    @Override
+    public boolean needsFiltering(Index.Group indexGroup)
+    {
+        return restrictions.needsFiltering(indexGroup);
     }
 
     @Override
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/TokenRestriction.java b/src/java/org/apache/cassandra/cql3/restrictions/TokenRestriction.java
index d7477fbff9..215f3237e8 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/TokenRestriction.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/TokenRestriction.java
@@ -22,6 +22,7 @@ import java.util.*;
 
 import com.google.common.base.Joiner;
 
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.cql3.QueryOptions;
@@ -123,11 +124,17 @@ public abstract class TokenRestriction implements PartitionKeyRestrictions
     }
 
     @Override
-    public void addRowFilterTo(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
+    public void addToRowFilter(RowFilter filter, IndexRegistry indexRegistry, QueryOptions options)
     {
         throw new UnsupportedOperationException("Index expression cannot be created for token restriction");
     }
 
+    @Override
+    public boolean needsFiltering(Index.Group indexGroup)
+    {
+        return false;
+    }
+
     @Override
     public final boolean isEmpty()
     {
diff --git a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
index 4c43cf8d4f..a8986ee130 100644
--- a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java
@@ -1174,7 +1174,7 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement
                     orderingComparator = Collections.reverseOrder(orderingComparator);
             }
 
-            checkNeedsFiltering(restrictions);
+            checkNeedsFiltering(table, restrictions);
 
             return new SelectStatement(table,
                                        bindVariables,
@@ -1483,16 +1483,15 @@ public class SelectStatement implements CQLStatement.SingleKeyspaceCqlStatement
         }
 
         /** If ALLOW FILTERING was not specified, this verifies that it is not needed */
-        private void checkNeedsFiltering(StatementRestrictions restrictions) throws InvalidRequestException
+        private void checkNeedsFiltering(TableMetadata table, StatementRestrictions restrictions) throws InvalidRequestException
         {
             // non-key-range non-indexed queries cannot involve filtering underneath
             if (!parameters.allowFiltering && (restrictions.isKeyRange() || restrictions.usesSecondaryIndexing()))
             {
-                // We will potentially filter data if either:
-                //  - Have more than one IndexExpression
-                //  - Have no index expression and the row filter is not the identity
+                // We will potentially filter data if the row filter is not the identity and there isn't any index group
+                // supporting all the expressions in the filter.
                 if (restrictions.requiresAllowFilteringIfNotSpecified())
-                    checkFalse(restrictions.needFiltering(), StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
+                    checkFalse(restrictions.needFiltering(table), StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
             }
         }
 
diff --git a/src/java/org/apache/cassandra/index/Index.java b/src/java/org/apache/cassandra/index/Index.java
index 152606069e..64560e098a 100644
--- a/src/java/org/apache/cassandra/index/Index.java
+++ b/src/java/org/apache/cassandra/index/Index.java
@@ -220,7 +220,7 @@ public interface Index
     {
         return INDEX_BUILDER_SUPPORT;
     }
-    
+
     /**
      * Same as {@code getBuildTaskSupport} but can be overloaded with a specific 'recover' logic different than the index building one
      */
@@ -228,7 +228,7 @@ public interface Index
     {
         return getBuildTaskSupport();
     }
-    
+
     /**
      * Returns the type of operations supported by the index in case its building has failed and it's needing recovery.
      *
@@ -277,9 +277,9 @@ public interface Index
 
     /**
      * If the index implementation uses a local table to store its index data, this method should return a
-     * handle to it. If not, an empty {@link Optional} should be returned. This exists to support legacy 
+     * handle to it. If not, an empty {@link Optional} should be returned. This exists to support legacy
      * implementations, and should always be empty for indexes not belonging to a {@link SingletonIndexGroup}.
-     * 
+     *
      * @return an Optional referencing the Index's backing storage table if it has one, or Optional.empty() if not.
      */
     public Optional<ColumnFamilyStore> getBackingTable();
@@ -389,6 +389,17 @@ public interface Index
      */
     public boolean supportsExpression(ColumnMetadata column, Operator operator);
 
+    /**
+     * Returns whether this index does any kind of filtering when the query has multiple contains expressions, assuming
+     * that each of those expressions are supported as defined by {@link #supportsExpression(ColumnMetadata, Operator)}.
+     *
+     * @return {@code true} if this index uses filtering on multiple contains expressions, {@code false} otherwise
+     */
+    default boolean filtersMultipleContains()
+    {
+        return true;
+    }
+
     /**
      * If the index supports custom search expressions using the
      * {@code}SELECT * FROM table WHERE expr(index_name, expression){@code} syntax, this
@@ -636,7 +647,7 @@ public interface Index
          * Replica filtering protection may fetch data that doesn't match query conditions.
          *
          * On coordinator, we need to filter the replicas' responses again.
-         * 
+         *
          * This will not be called if {@link QueryPlan#supportsReplicaFilteringProtection(RowFilter)} returns false.
          *
          * @return filtered response that satisfied query conditions
diff --git a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java
index 8bb99fa04d..1d6c94ff31 100644
--- a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java
+++ b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java
@@ -480,6 +480,12 @@ public class StorageAttachedIndex implements Index
         return dependsOn(column) && indexContext.supports(operator);
     }
 
+    @Override
+    public boolean filtersMultipleContains()
+    {
+        return false;
+    }
+
     @Override
     public AbstractType<?> customExpressionValueType()
     {
diff --git a/test/unit/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictionsTest.java b/test/unit/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictionsTest.java
index 029729fe78..bf8e1e4955 100644
--- a/test/unit/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictionsTest.java
+++ b/test/unit/org/apache/cassandra/cql3/restrictions/ClusteringColumnRestrictionsTest.java
@@ -54,7 +54,7 @@ public class ClusteringColumnRestrictionsTest
     {
         TableMetadata tableMetadata = newTableMetadata(Sort.ASC);
 
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
+        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -76,8 +76,7 @@ public class ClusteringColumnRestrictionsTest
         ByteBuffer clustering_0 = ByteBufferUtil.bytes(1);
         Restriction eq = newSingleEq(tableMetadata, 0, clustering_0);
 
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(eq);
+        ClusteringColumnRestrictions restrictions =  restrictions(tableMetadata, eq);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -99,8 +98,7 @@ public class ClusteringColumnRestrictionsTest
         ByteBuffer clustering_0 = ByteBufferUtil.bytes(1);
         Restriction eq = newSingleEq(tableMetadata, 0, clustering_0);
 
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(eq);
+        ClusteringColumnRestrictions restrictions =  restrictions(tableMetadata, eq);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -124,9 +122,7 @@ public class ClusteringColumnRestrictionsTest
         TableMetadata tableMetadata = newTableMetadata(Sort.ASC, Sort.ASC);
 
         Restriction in = newSingleIN(tableMetadata, 0, value1, value2, value3);
-
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(in);
+        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, in);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(3, bounds.size());
@@ -153,8 +149,7 @@ public class ClusteringColumnRestrictionsTest
         ByteBuffer value2 = ByteBufferUtil.bytes(2);
 
         Restriction slice = newSingleSlice(tableMetadata, 0, Bound.START, false, value1);
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, slice);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -165,8 +160,7 @@ public class ClusteringColumnRestrictionsTest
         assertEmptyEnd(get(bounds, 0));
 
         slice = newSingleSlice(tableMetadata, 0, Bound.START, true, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -177,8 +171,7 @@ public class ClusteringColumnRestrictionsTest
         assertEmptyEnd(get(bounds, 0));
 
         slice = newSingleSlice(tableMetadata, 0, Bound.END, true, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -189,8 +182,7 @@ public class ClusteringColumnRestrictionsTest
         assertEndBound(get(bounds, 0), true, value1);
 
         slice = newSingleSlice(tableMetadata, 0, Bound.END, false, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -202,8 +194,7 @@ public class ClusteringColumnRestrictionsTest
 
         slice = newSingleSlice(tableMetadata, 0, Bound.START, false, value1);
         Restriction slice2 = newSingleSlice(tableMetadata, 0, Bound.END, false, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -215,8 +206,7 @@ public class ClusteringColumnRestrictionsTest
 
         slice = newSingleSlice(tableMetadata, 0, Bound.START, true, value1);
         slice2 = newSingleSlice(tableMetadata, 0, Bound.END, true, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -239,8 +229,7 @@ public class ClusteringColumnRestrictionsTest
         ByteBuffer value2 = ByteBufferUtil.bytes(2);
 
         Restriction slice = newSingleSlice(tableMetadata, 0, Bound.START, false, value1);
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, slice);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -251,8 +240,7 @@ public class ClusteringColumnRestrictionsTest
         assertEndBound(get(bounds, 0), false, value1);
 
         slice = newSingleSlice(tableMetadata, 0, Bound.START, true, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -263,8 +251,7 @@ public class ClusteringColumnRestrictionsTest
         assertEndBound(get(bounds, 0), true, value1);
 
         slice = newSingleSlice(tableMetadata, 0, Bound.END, true, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -275,8 +262,7 @@ public class ClusteringColumnRestrictionsTest
         assertEmptyEnd(get(bounds, 0));
 
         slice = newSingleSlice(tableMetadata, 0, Bound.END, false, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -288,8 +274,7 @@ public class ClusteringColumnRestrictionsTest
 
         slice = newSingleSlice(tableMetadata, 0, Bound.START, false, value1);
         Restriction slice2 = newSingleSlice(tableMetadata, 0, Bound.END, false, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -301,8 +286,7 @@ public class ClusteringColumnRestrictionsTest
 
         slice = newSingleSlice(tableMetadata, 0, Bound.START, true, value1);
         slice2 = newSingleSlice(tableMetadata, 0, Bound.END, true, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -326,8 +310,7 @@ public class ClusteringColumnRestrictionsTest
         ByteBuffer value3 = ByteBufferUtil.bytes(3);
         Restriction eq = newSingleEq(tableMetadata, 0, value1);
         Restriction in = newSingleIN(tableMetadata, 1, value1, value2, value3);
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(eq).mergeWith(in);
+        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, eq, in);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(3, bounds.size());
@@ -357,8 +340,7 @@ public class ClusteringColumnRestrictionsTest
         Restriction eq = newSingleEq(tableMetadata, 0, value3);
 
         Restriction slice = newSingleSlice(tableMetadata, 1, Bound.START, false, value1);
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(eq).mergeWith(slice);
+        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, eq, slice);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -369,8 +351,7 @@ public class ClusteringColumnRestrictionsTest
         assertEndBound(get(bounds, 0), true, value3);
 
         slice = newSingleSlice(tableMetadata, 1, Bound.START, true, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(eq).mergeWith(slice);
+        restrictions = restrictions(tableMetadata, eq, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -381,8 +362,7 @@ public class ClusteringColumnRestrictionsTest
         assertEndBound(get(bounds, 0), true, value3);
 
         slice = newSingleSlice(tableMetadata, 1, Bound.END, true, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(eq).mergeWith(slice);
+        restrictions =  restrictions(tableMetadata, eq, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -393,8 +373,7 @@ public class ClusteringColumnRestrictionsTest
         assertEndBound(get(bounds, 0), true, value3, value1);
 
         slice = newSingleSlice(tableMetadata, 1, Bound.END, false, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(eq).mergeWith(slice);
+        restrictions =  restrictions(tableMetadata, eq, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -406,8 +385,7 @@ public class ClusteringColumnRestrictionsTest
 
         slice = newSingleSlice(tableMetadata, 1, Bound.START, false, value1);
         Restriction slice2 = newSingleSlice(tableMetadata, 1, Bound.END, false, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(eq).mergeWith(slice).mergeWith(slice2);
+        restrictions =  restrictions(tableMetadata, eq, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -419,8 +397,7 @@ public class ClusteringColumnRestrictionsTest
 
         slice = newSingleSlice(tableMetadata, 1, Bound.START, true, value1);
         slice2 = newSingleSlice(tableMetadata, 1, Bound.END, true, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(eq).mergeWith(slice).mergeWith(slice2);
+        restrictions =  restrictions(tableMetadata, eq, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -442,8 +419,7 @@ public class ClusteringColumnRestrictionsTest
         ByteBuffer value1 = ByteBufferUtil.bytes(1);
         ByteBuffer value2 = ByteBufferUtil.bytes(2);
         Restriction eq = newMultiEq(tableMetadata, 0, value1, value2);
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(eq);
+        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, eq);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -466,8 +442,7 @@ public class ClusteringColumnRestrictionsTest
         ByteBuffer value2 = ByteBufferUtil.bytes(2);
         ByteBuffer value3 = ByteBufferUtil.bytes(3);
         Restriction in = newMultiIN(tableMetadata, 0, asList(value1, value2), asList(value2, value3));
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(in);
+        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, in);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -493,8 +468,7 @@ public class ClusteringColumnRestrictionsTest
         ByteBuffer value2 = ByteBufferUtil.bytes(2);
 
         Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1);
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, slice);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -505,8 +479,7 @@ public class ClusteringColumnRestrictionsTest
         assertEmptyEnd(get(bounds, 0));
 
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -517,8 +490,7 @@ public class ClusteringColumnRestrictionsTest
         assertEmptyEnd(get(bounds, 0));
 
         slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -529,8 +501,7 @@ public class ClusteringColumnRestrictionsTest
         assertEndBound(get(bounds, 0), true, value1);
 
         slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -542,8 +513,7 @@ public class ClusteringColumnRestrictionsTest
 
         slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1);
         Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -555,8 +525,7 @@ public class ClusteringColumnRestrictionsTest
 
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1);
         slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -580,8 +549,7 @@ public class ClusteringColumnRestrictionsTest
         ByteBuffer value2 = ByteBufferUtil.bytes(2);
 
         Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1);
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        ClusteringColumnRestrictions restrictions =  restrictions(tableMetadata, slice);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -592,8 +560,7 @@ public class ClusteringColumnRestrictionsTest
         assertEndBound(get(bounds, 0), false, value1);
 
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -604,8 +571,7 @@ public class ClusteringColumnRestrictionsTest
         assertEndBound(get(bounds, 0), true, value1);
 
         slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -616,8 +582,7 @@ public class ClusteringColumnRestrictionsTest
         assertEmptyEnd(get(bounds, 0));
 
         slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -629,8 +594,7 @@ public class ClusteringColumnRestrictionsTest
 
         slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1);
         Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -642,8 +606,7 @@ public class ClusteringColumnRestrictionsTest
 
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1);
         slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -667,8 +630,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1) > (1, 2)
         Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        ClusteringColumnRestrictions restrictions =  restrictions(tableMetadata, slice);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -680,8 +642,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1) >= (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -693,8 +654,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1) <= (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -706,8 +666,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1) < (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -720,8 +679,7 @@ public class ClusteringColumnRestrictionsTest
         // (clustering_0, clustering1) > (1, 2) AND (clustering_0) < (2)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
         Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -734,8 +692,7 @@ public class ClusteringColumnRestrictionsTest
         // (clustering_0, clustering1) >= (1, 2) AND (clustering_0, clustering1) <= (2, 1)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
         slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value2, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -759,8 +716,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1) > (1, 2)
         Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        ClusteringColumnRestrictions restrictions =  restrictions(tableMetadata, slice);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -772,8 +728,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1) >= (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -785,8 +740,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1) <= (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -798,8 +752,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1) < (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -813,8 +766,7 @@ public class ClusteringColumnRestrictionsTest
         // (clustering_0, clustering1) > (1, 2) AND (clustering_0) < (2)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
         Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -827,8 +779,7 @@ public class ClusteringColumnRestrictionsTest
         // (clustering_0, clustering1) >= (1, 2) AND (clustering_0, clustering1) <= (2, 1)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
         slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value2, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -853,8 +804,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1) > (1, 2)
         Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        ClusteringColumnRestrictions restrictions =  restrictions(tableMetadata, slice);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -868,8 +818,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1) >= (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -883,8 +832,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1) <= (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -898,8 +846,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1) < (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -914,8 +861,7 @@ public class ClusteringColumnRestrictionsTest
         // (clustering_0, clustering1) > (1, 2) AND (clustering_0) < (2)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
         Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -930,8 +876,7 @@ public class ClusteringColumnRestrictionsTest
         // (clustering_0) > (1) AND (clustering_0, clustering1) < (2, 1)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1);
         slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -946,8 +891,7 @@ public class ClusteringColumnRestrictionsTest
         // (clustering_0, clustering1) >= (1, 2) AND (clustering_0, clustering1) <= (2, 1)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
         slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value2, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(3, bounds.size());
@@ -976,8 +920,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1) > (1, 2)
         Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        ClusteringColumnRestrictions restrictions =  restrictions(tableMetadata, slice);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -991,8 +934,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1) >= (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1006,8 +948,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1) <= (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1021,8 +962,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1) < (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1037,8 +977,7 @@ public class ClusteringColumnRestrictionsTest
         // (clustering_0, clustering1) > (1, 2) AND (clustering_0) < (2)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2);
         Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1053,8 +992,7 @@ public class ClusteringColumnRestrictionsTest
         // (clustering_0, clustering1) >= (1, 2) AND (clustering_0, clustering1) <= (2, 1)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
         slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value2, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(3, bounds.size());
@@ -1085,8 +1023,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1, clustering_2, clustering_3) > (1, 2, 3, 4)
         Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2, value3, value4);
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        ClusteringColumnRestrictions restrictions =  restrictions(tableMetadata, slice);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1101,9 +1038,7 @@ public class ClusteringColumnRestrictionsTest
         // clustering_0 = 1 AND (clustering_1, clustering_2, clustering_3) > (2, 3, 4)
         Restriction eq = newSingleEq(tableMetadata, 0, value1);
         slice = newMultiSlice(tableMetadata, 1, Bound.START, false, value2, value3, value4);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
-        restrictions = restrictions.mergeWith(eq);
+        restrictions = restrictions(tableMetadata, slice, eq);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1118,9 +1053,7 @@ public class ClusteringColumnRestrictionsTest
         // clustering_0 IN (1, 2) AND (clustering_1, clustering_2, clustering_3) > (2, 3, 4)
         Restriction in = newSingleIN(tableMetadata, 0, value1, value2);
         slice = newMultiSlice(tableMetadata, 1, Bound.START, false, value2, value3, value4);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
-        restrictions = restrictions.mergeWith(in);
+        restrictions = restrictions(tableMetadata, slice, in);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(4, bounds.size());
@@ -1138,8 +1071,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1) >= (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1151,8 +1083,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1, clustering_2, clustering_3) >= (1, 2, 3, 4)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2, value3, value4);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1166,8 +1097,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1, clustering_2, clustering_3) <= (1, 2, 3, 4)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1, value2, value3, value4);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1181,8 +1111,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1, clustering_2, clustering_3) < (1, 2, 3, 4)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1, value2, value3, value4);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1197,8 +1126,7 @@ public class ClusteringColumnRestrictionsTest
         // (clustering_0, clustering1, clustering_2, clustering_3) > (1, 2, 3, 4) AND (clustering_0, clustering_1) < (2, 3)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2, value3, value4);
         Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2, value3);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1213,8 +1141,7 @@ public class ClusteringColumnRestrictionsTest
         // (clustering_0, clustering1, clustering_2, clustering_3) >= (1, 2, 3, 4) AND (clustering_0, clustering1, clustering_2, clustering_3) <= (4, 3, 2, 1)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2, value3, value4);
         slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value4, value3, value2, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(3, bounds.size());
@@ -1245,8 +1172,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1, clustering_2, clustering_3) > (1, 2, 3, 4)
         Restriction slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2, value3, value4);
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, slice);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(4, bounds.size());
@@ -1266,9 +1192,7 @@ public class ClusteringColumnRestrictionsTest
         // clustering_0 = 1 AND (clustering_1, clustering_2, clustering_3) > (2, 3, 4)
         Restriction eq = newSingleEq(tableMetadata, 0, value1);
         slice = newMultiSlice(tableMetadata, 1, Bound.START, false, value2, value3, value4);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
-        restrictions = restrictions.mergeWith(eq);
+        restrictions = restrictions(tableMetadata, slice, eq);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(3, bounds.size());
@@ -1284,8 +1208,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1) >= (1, 2)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1299,8 +1222,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1, clustering_2, clustering_3) >= (1, 2, 3, 4)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2, value3, value4);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(4, bounds.size());
@@ -1318,8 +1240,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1, clustering_2, clustering_3) <= (1, 2, 3, 4)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, true, value1, value2, value3, value4);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(4, bounds.size());
@@ -1337,8 +1258,7 @@ public class ClusteringColumnRestrictionsTest
 
         // (clustering_0, clustering1, clustering_2, clustering_3) < (1, 2, 3, 4)
         slice = newMultiSlice(tableMetadata, 0, Bound.END, false, value1, value2, value3, value4);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice);
+        restrictions = restrictions(tableMetadata, slice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(4, bounds.size());
@@ -1357,8 +1277,7 @@ public class ClusteringColumnRestrictionsTest
         // (clustering_0, clustering1, clustering_2, clustering_3) > (1, 2, 3, 4) AND (clustering_0, clustering_1) < (2, 3)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, false, value1, value2, value3, value4);
         Restriction slice2 = newMultiSlice(tableMetadata, 0, Bound.END, false, value2, value3);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(5, bounds.size());
@@ -1379,8 +1298,7 @@ public class ClusteringColumnRestrictionsTest
         // (clustering_0, clustering1, clustering_2, clustering_3) >= (1, 2, 3, 4) AND (clustering_0, clustering1, clustering_2, clustering_3) <= (4, 3, 2, 1)
         slice = newMultiSlice(tableMetadata, 0, Bound.START, true, value1, value2, value3, value4);
         slice2 = newMultiSlice(tableMetadata, 0, Bound.END, true, value4, value3, value2, value1);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(slice).mergeWith(slice2);
+        restrictions = restrictions(tableMetadata, slice, slice2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(7, bounds.size());
@@ -1419,8 +1337,7 @@ public class ClusteringColumnRestrictionsTest
         // clustering_0 = 1 AND (clustering_1, clustering_2) = (2, 3)
         Restriction singleEq = newSingleEq(tableMetadata, 0, value1);
         Restriction multiEq = newMultiEq(tableMetadata, 1, value2, value3);
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(singleEq).mergeWith(multiEq);
+        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, singleEq, multiEq);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1434,8 +1351,7 @@ public class ClusteringColumnRestrictionsTest
         singleEq = newSingleEq(tableMetadata, 0, value1);
         Restriction singleEq2 = newSingleEq(tableMetadata, 1, value2);
         multiEq = newMultiEq(tableMetadata, 2, value3, value4);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(singleEq).mergeWith(singleEq2).mergeWith(multiEq);
+        restrictions = restrictions(tableMetadata, singleEq, singleEq2, multiEq);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1448,8 +1364,7 @@ public class ClusteringColumnRestrictionsTest
         // (clustering_0, clustering_1) = (1, 2) AND clustering_2 = 3
         singleEq = newSingleEq(tableMetadata, 2, value3);
         multiEq = newMultiEq(tableMetadata, 0, value1, value2);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(singleEq).mergeWith(multiEq);
+        restrictions = restrictions(tableMetadata, singleEq, multiEq);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1463,8 +1378,7 @@ public class ClusteringColumnRestrictionsTest
         singleEq = newSingleEq(tableMetadata, 0, value1);
         singleEq2 = newSingleEq(tableMetadata, 3, value4);
         multiEq = newMultiEq(tableMetadata, 1, value2, value3);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(singleEq).mergeWith(multiEq).mergeWith(singleEq2);
+        restrictions = restrictions(tableMetadata, singleEq, multiEq, singleEq2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1492,8 +1406,7 @@ public class ClusteringColumnRestrictionsTest
         // clustering_0 = 1 AND (clustering_1, clustering_2) IN ((2, 3), (4, 5))
         Restriction singleEq = newSingleEq(tableMetadata, 0, value1);
         Restriction multiIN = newMultiIN(tableMetadata, 1, asList(value2, value3), asList(value4, value5));
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(singleEq).mergeWith(multiIN);
+        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, singleEq, multiIN);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1508,8 +1421,7 @@ public class ClusteringColumnRestrictionsTest
         // clustering_0 = 1 AND (clustering_1, clustering_2) IN ((2, 3))
         singleEq = newSingleEq(tableMetadata, 0, value1);
         multiIN = newMultiIN(tableMetadata, 1, asList(value2, value3));
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(multiIN).mergeWith(singleEq);
+        restrictions = restrictions(tableMetadata, multiIN, singleEq);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1523,8 +1435,7 @@ public class ClusteringColumnRestrictionsTest
         singleEq = newSingleEq(tableMetadata, 0, value1);
         Restriction singleEq2 = newSingleEq(tableMetadata, 1, value5);
         multiIN = newMultiIN(tableMetadata, 2, asList(value2, value3), asList(value4, value5));
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(singleEq).mergeWith(multiIN).mergeWith(singleEq2);
+        restrictions = restrictions(tableMetadata, singleEq, multiIN, singleEq2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1555,8 +1466,7 @@ public class ClusteringColumnRestrictionsTest
         // clustering_0 = 1 AND (clustering_1, clustering_2) > (2, 3)
         Restriction singleEq = newSingleEq(tableMetadata, 0, value1);
         Restriction multiSlice = newMultiSlice(tableMetadata, 1, Bound.START, false, value2, value3);
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(singleEq).mergeWith(multiSlice);
+        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, singleEq, multiSlice);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1570,8 +1480,7 @@ public class ClusteringColumnRestrictionsTest
         singleEq = newSingleEq(tableMetadata, 0, value1);
         multiSlice = newMultiSlice(tableMetadata, 1, Bound.START, false, value2, value3);
         Restriction multiSlice2 = newMultiSlice(tableMetadata, 1, Bound.END, false, value4);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(multiSlice2).mergeWith(singleEq).mergeWith(multiSlice);
+        restrictions = restrictions(tableMetadata, multiSlice2, singleEq, multiSlice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1585,8 +1494,7 @@ public class ClusteringColumnRestrictionsTest
         singleEq = newSingleEq(tableMetadata, 0, value1);
         multiSlice = newMultiSlice(tableMetadata, 1, Bound.START, true, value2, value3);
         multiSlice2 = newMultiSlice(tableMetadata, 1, Bound.END, true, value4, value5);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(multiSlice2).mergeWith(singleEq).mergeWith(multiSlice);
+        restrictions = restrictions(tableMetadata, multiSlice2, singleEq, multiSlice);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1613,8 +1521,7 @@ public class ClusteringColumnRestrictionsTest
         // (clustering_0, clustering_1) = (1, 2) AND clustering_2 > 3
         Restriction multiEq = newMultiEq(tableMetadata, 0, value1, value2);
         Restriction singleSlice = newSingleSlice(tableMetadata, 2, Bound.START, false, value3);
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(multiEq).mergeWith(singleSlice);
+        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, multiEq, singleSlice);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1639,8 +1546,7 @@ public class ClusteringColumnRestrictionsTest
         // (clustering_0, clustering_1) = (1, 2) AND (clustering_2, clustering_3) > (3, 4)
         Restriction multiEq = newMultiEq(tableMetadata, 0, value1, value2);
         Restriction multiSlice = newMultiSlice(tableMetadata, 2, Bound.START, false, value3, value4);
-        ClusteringColumnRestrictions restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(multiEq).mergeWith(multiSlice);
+        ClusteringColumnRestrictions restrictions = restrictions(tableMetadata, multiEq, multiSlice);
 
         SortedSet<ClusteringBound<?>> bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1653,8 +1559,7 @@ public class ClusteringColumnRestrictionsTest
         // (clustering_0, clustering_1) = (1, 2) AND (clustering_2, clustering_3) IN ((3, 4), (4, 5))
         multiEq = newMultiEq(tableMetadata, 0, value1, value2);
         Restriction multiIN = newMultiIN(tableMetadata, 2, asList(value3, value4), asList(value4, value5));
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(multiEq).mergeWith(multiIN);
+        restrictions = restrictions(tableMetadata, multiEq, multiIN);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(2, bounds.size());
@@ -1669,8 +1574,7 @@ public class ClusteringColumnRestrictionsTest
         // (clustering_0, clustering_1) = (1, 2) AND (clustering_2, clustering_3) = (3, 4)
         multiEq = newMultiEq(tableMetadata, 0, value1, value2);
         Restriction multiEq2 = newMultiEq(tableMetadata, 2, value3, value4);
-        restrictions = new ClusteringColumnRestrictions(tableMetadata);
-        restrictions = restrictions.mergeWith(multiEq).mergeWith(multiEq2);
+        restrictions = restrictions(tableMetadata, multiEq, multiEq2);
 
         bounds = restrictions.boundsAsClustering(Bound.START, QueryOptions.DEFAULT);
         assertEquals(1, bounds.size());
@@ -1733,11 +1637,10 @@ public class ClusteringColumnRestrictionsTest
         for (int i = 0, m = elements.length; i < m; i++)
         {
             ByteBuffer element = elements[i];
-            assertTrue(String.format("the element %s of the bound is not the expected one: expected %s but was %s",
-                                     i,
-                                     ByteBufferUtil.toInt(element),
-                                     ByteBufferUtil.toInt(bound.bufferAt(i))),
-                       bound.bufferAt(i).equals(element));
+            assertEquals(String.format("the element %s of the bound is not the expected one: expected %s but was %s",
+                                       i,
+                                       ByteBufferUtil.toInt(element),
+                                       ByteBufferUtil.toInt(bound.bufferAt(i))), bound.bufferAt(i), element);
         }
     }
 
@@ -1913,8 +1816,16 @@ public class ClusteringColumnRestrictionsTest
     {
         return Iterables.get(set, i);
     }
+    
+    private static ClusteringColumnRestrictions restrictions(TableMetadata table, Restriction... restrictions)
+    {
+        ClusteringColumnRestrictions clusteringColumnRestrictions = new ClusteringColumnRestrictions(table, false);
+        for (Restriction restriction : restrictions)
+            clusteringColumnRestrictions = clusteringColumnRestrictions.mergeWith(restriction, null);
+        return clusteringColumnRestrictions;
+    }
 
-    private static enum Sort
+    private enum Sort
     {
         ASC,
         DESC;
diff --git a/test/unit/org/apache/cassandra/index/CustomIndexTest.java b/test/unit/org/apache/cassandra/index/CustomIndexTest.java
index 2a4486f7d4..357c3efc32 100644
--- a/test/unit/org/apache/cassandra/index/CustomIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/CustomIndexTest.java
@@ -37,6 +37,7 @@ import com.datastax.driver.core.exceptions.QueryValidationException;
 import org.apache.cassandra.Util;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.cql3.ColumnIdentifier;
+import org.apache.cassandra.cql3.Operator;
 import org.apache.cassandra.cql3.restrictions.IndexRestrictions;
 import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
 import org.apache.cassandra.cql3.statements.schema.IndexTarget;
@@ -57,6 +58,7 @@ import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableFlushObserver;
 import org.apache.cassandra.index.transactions.IndexTransaction;
+import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.Indexes;
 import org.apache.cassandra.schema.TableMetadata;
@@ -370,7 +372,7 @@ public class CustomIndexTest extends CQLTester
         assertInvalidMessage(String.format(IndexRestrictions.INDEX_NOT_FOUND, indexName, currentTableMetadata().toString()),
                              String.format("SELECT * FROM %%s WHERE expr(%s, 'foo bar baz')", indexName));
 
-        createIndex(String.format("CREATE CUSTOM INDEX %s ON %%s(c) USING '%s'", indexName, StubIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX %s ON %%s(c) USING '%s'", indexName, ColumnTargetedIndex.class.getName()));
 
         assertInvalidThrowMessage(Optional.of(ProtocolVersion.CURRENT),
                                   String.format(IndexRestrictions.INDEX_NOT_FOUND, "no_such_index", currentTableMetadata().toString()),
@@ -390,7 +392,7 @@ public class CustomIndexTest extends CQLTester
                                                 indexName));
 
         // multiple expressions on different indexes
-        createIndex(String.format("CREATE CUSTOM INDEX other_custom_index ON %%s(d) USING '%s'", StubIndex.class.getName()));
+        createIndex(String.format("CREATE CUSTOM INDEX other_custom_index ON %%s(d) USING '%s'", ColumnTargetedIndex.class.getName()));
         assertInvalidThrowMessage(Optional.of(ProtocolVersion.CURRENT),
                                   IndexRestrictions.MULTIPLE_EXPRESSIONS,
                                   QueryValidationException.class,
@@ -404,6 +406,26 @@ public class CustomIndexTest extends CQLTester
         assertRows(execute(String.format("SELECT * FROM %%s WHERE expr(%s, 'foo') AND d=0 ALLOW FILTERING", indexName)), row);
     }
 
+    /**
+     * A {@link StubIndex} that only supports expressions on its target column.
+     */
+    public static final class ColumnTargetedIndex extends StubIndex
+    {
+        private final ColumnMetadata indexedColumn;
+
+        public ColumnTargetedIndex(ColumnFamilyStore baseCfs, IndexMetadata metadata)
+        {
+            super(baseCfs, metadata);
+            indexedColumn = TargetParser.parse(baseCfs.metadata(), metadata).left;
+        }
+
+        @Override
+        public boolean supportsExpression(ColumnMetadata column, Operator operator)
+        {
+            return column.equals(indexedColumn) && super.supportsExpression(column, operator);
+        }
+    }
+
     @Test
     public void customIndexDoesntSupportCustomExpressions() throws Throwable
     {
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/AllowFilteringTest.java b/test/unit/org/apache/cassandra/index/sai/cql/AllowFilteringTest.java
new file mode 100644
index 0000000000..13b0f6148b
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/AllowFilteringTest.java
@@ -0,0 +1,314 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+
+import static java.lang.String.format;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Tests that {@code ALLOW FILTERING} is required only if needed.
+ */
+public class AllowFilteringTest extends SAITester
+{
+    @Test
+    public void testAllowFilteringOnFirstClusteringKeyColumn() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k1 int, k2 int, c1 int, c2 int, c3 int, v1 int, " +
+                    "PRIMARY KEY ((k1, k2), c1, c2, c3))");
+        createIndex(format("CREATE CUSTOM INDEX ON %%s(c1) USING '%s'", StorageAttachedIndex.class.getName()));
+        waitForIndexQueryable();
+
+        // with only index restrictions
+        test("SELECT * FROM %s WHERE c1=0", false);
+        test("SELECT * FROM %s WHERE c1>0", false);
+        test("SELECT * FROM %s WHERE c1>0 AND c1<1", false);
+
+        // with additional simple filtering restrictions
+        test("SELECT * FROM %s WHERE c1=0 AND k1=0", true);
+        test("SELECT * FROM %s WHERE c1=0 AND k2=0", true);
+        test("SELECT * FROM %s WHERE c1=0 AND c2=0", true);
+        test("SELECT * FROM %s WHERE c1=0 AND c3=0", true);
+        test("SELECT * FROM %s WHERE c1=0 AND v1=0", true);
+
+        // with token restrictions
+        test("SELECT * FROM %s WHERE c1=0 AND token(k1, k2) = token(0, 0)", false);
+        test("SELECT * FROM %s WHERE c1=0 AND token(k1, k2) > token(0, 0)", false);
+        test("SELECT * FROM %s WHERE c1=0 AND token(k1, k2) > token(0, 0) AND token(k1, k2) <= token(1, 1)", false);
+
+        // with restriction on partition key
+        test("SELECT * FROM %s WHERE c1=0 AND k1=0 AND k2=0", false);
+        test("SELECT * FROM %s WHERE c1=0 AND k1=0 AND k2=0 AND v1=0", true);
+        test("SELECT * FROM %s WHERE c1=0 AND k1=0 AND k2=0 AND c3=0", true);
+
+        // with restriction on partition key and clustering key prefix
+        test("SELECT * FROM %s WHERE c1=0 AND k1=0 AND k2=0 AND c2=0", false);
+        test("SELECT * FROM %s WHERE c1=0 AND k1=0 AND k2=0 AND c2=0 AND v1=0", true);
+        test("SELECT * FROM %s WHERE c1=0 AND k1=0 AND k2=0 AND c2=0 AND c3>0", false);
+        test("SELECT * FROM %s WHERE c1=0 AND k1=0 AND k2=0 AND c2=0 AND c3>0 AND v1=0", true);
+
+        // with restriction on partition key and full clustering key
+        test("SELECT * FROM %s WHERE c1=0 AND k1=0 AND k2=0 AND c2=0 AND c3=0", false);
+        test("SELECT * FROM %s WHERE c1=0 AND k1=0 AND k2=0 AND c2=0 AND c3=0 AND v1=0", true);
+
+        // with restriction on partition key and full clustering key, multicolumn format
+        test("SELECT * FROM %s WHERE k1=0 AND k2=0 AND (c1, c2, c3) = (0, 0, 0)", false);
+        test("SELECT * FROM %s WHERE k1=0 AND k2=0 AND (c1, c2, c3) = (0, 0, 0) AND v1=0", true);
+    }
+
+    @Test
+    public void testAllowFilteringOnNotFirstClusteringKeyColumn() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k1 int, k2 int, c1 int, c2 int, c3 int, c4 int, v1 int, " +
+                    "PRIMARY KEY ((k1, k2), c1, c2, c3, c4))");
+        createIndex(format("CREATE CUSTOM INDEX ON %%s(c3) USING '%s'", StorageAttachedIndex.class.getName()));
+        waitForIndexQueryable();
+
+        // with only index restrictions
+        test("SELECT * FROM %s WHERE c3=0", false);
+        test("SELECT * FROM %s WHERE c3>0", false);
+        test("SELECT * FROM %s WHERE c3>0 AND c3<1", false);
+
+        // with additional simple filtering restrictions
+        test("SELECT * FROM %s WHERE c3=0 AND k1=0", true);
+        test("SELECT * FROM %s WHERE c3=0 AND k2=0", true);
+        test("SELECT * FROM %s WHERE c3=0 AND c1=0", true);
+        test("SELECT * FROM %s WHERE c3=0 AND c2=0", true);
+        test("SELECT * FROM %s WHERE c3=0 AND c4=0", true);
+        test("SELECT * FROM %s WHERE c3=0 AND v1=0", true);
+
+        // with token restrictions
+        test("SELECT * FROM %s WHERE c3=0 AND token(k1, k2) = token(0, 0)", false);
+        test("SELECT * FROM %s WHERE c3=0 AND token(k1, k2) > token(0, 0)", false);
+        test("SELECT * FROM %s WHERE c3=0 AND token(k1, k2) > token(0, 0) AND token(k1, k2) <= token(1, 1)", false);
+
+        // with restriction on partition key
+        test("SELECT * FROM %s WHERE c3=0 AND k1=0 AND k2=0", false);
+        test("SELECT * FROM %s WHERE c3=0 AND k1=0 AND k2=0 AND v1=0", true);
+        test("SELECT * FROM %s WHERE c3=0 AND k1=0 AND k2=0 AND c2=0", true);
+        test("SELECT * FROM %s WHERE c3=0 AND k1=0 AND k2=0 AND c4=0", true);
+
+        // with restriction on partition key and clustering key prefix
+        test("SELECT * FROM %s WHERE c3=0 AND k1=0 AND k2=0 AND c1=0", true);
+        test("SELECT * FROM %s WHERE c3=0 AND k1=0 AND k2=0 AND c1=0 AND v1=0", true);
+
+        // with restriction on partition key and full clustering key
+        test("SELECT * FROM %s WHERE c3=0 AND k1=0 AND k2=0 AND c1=0 AND c2=0 AND c4=0", false);
+        test("SELECT * FROM %s WHERE c3=0 AND k1=0 AND k2=0 AND c1=0 AND c2=0 AND c4=0 AND v1=0", true);
+
+        // with restriction on partition key and full clustering key, multicolumn format
+        test("SELECT * FROM %s WHERE k1=0 AND k2=0 AND (c1, c2, c3, c4) = (0, 0, 0, 0)", false);
+        test("SELECT * FROM %s WHERE k1=0 AND k2=0 AND (c1, c2, c3, c4) = (0, 0, 0, 0) AND v1=0", true);
+    }
+
+    @Test
+    public void testAllowFilteringOnMultipleClusteringKeyColumns() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k1 int, k2 int, c1 int, c2 int, c3 int, c4 int, v1 int, " +
+                    "PRIMARY KEY ((k1, k2), c1, c2, c3, c4))");
+        createIndex(format("CREATE CUSTOM INDEX ON %%s(c2) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(format("CREATE CUSTOM INDEX ON %%s(c4) USING '%s'", StorageAttachedIndex.class.getName()));
+        waitForIndexQueryable();
+
+        // with only index restrictions
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0", false);
+        test("SELECT * FROM %s WHERE c2=0 AND c4>0", false);
+        test("SELECT * FROM %s WHERE c2=0 AND c4>0 AND c4<1", false);
+        test("SELECT * FROM %s WHERE c2>0 AND c4=0", false);
+        test("SELECT * FROM %s WHERE c2>0 AND c2<1 AND c4=0", false);
+        test("SELECT * FROM %s WHERE c2>0 AND c4>0", false);
+        test("SELECT * FROM %s WHERE c2>0 AND c2<1 AND c4>0 AND c4<1", false);
+
+        // with additional simple filtering restrictions
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND k1=0", true);
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND k2=0", true);
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND c1=0", true);
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND c3=0", true);
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0", true);
+
+        // with token restrictions
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND token(k1, k2) = token(0, 0)", false);
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND token(k1, k2) > token(0, 0)", false);
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND token(k1, k2) > token(0, 0) AND token(k1, k2) <= token(1, 1)", false);
+
+        // with restriction on partition key
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND k1=0 AND k2=0", false);
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND k1=0 AND k2=0 AND c3=0", true);
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND k1=0 AND k2=0 AND v1=0", true);
+
+        // with restriction on partition key and clustering key prefix
+        test("SELECT * FROM %s WHERE k1=2 AND k2=3 AND c1=4 AND c2=0 AND c4=1", true);
+        test("SELECT * FROM %s WHERE k1=0 AND k2=0 AND c1=0 AND c2=0 AND c4=0 AND v1=0", true);
+
+        // with restriction on partition key and full clustering key
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND k1=0 AND k2=0 AND c1=0 AND c3=0", false);
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND k1=0 AND k2=0 AND c1=0 AND c3=0 AND v1=0", true);
+
+        // with restriction on partition key and full clustering key, multicolumn format
+        test("SELECT * FROM %s WHERE k1=0 AND k2=0 AND (c1, c2, c3, c4) = (0, 0, 0, 0)", false);
+        test("SELECT * FROM %s WHERE k1=0 AND k2=0 AND (c1, c2, c3, c4) = (0, 0, 0, 0) AND v1=0", true);
+    }
+
+    @Test
+    public void testAllowFilteringOnSingleRegularColumn() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k1 int, k2 int, c1 int, c2 int, v1 int, v2 int, PRIMARY KEY ((k1, k2), c1, c2))");
+        createIndex(format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName()));
+        waitForIndexQueryable();
+
+        // with only index restrictions
+        test("SELECT * FROM %s WHERE v1=0", false);
+        test("SELECT * FROM %s WHERE v1>0", false);
+        test("SELECT * FROM %s WHERE v1>0 AND v1<1", false);
+
+        // with additional simple filtering restrictions
+        test("SELECT * FROM %s WHERE v1=0 AND k1=0", true);
+        test("SELECT * FROM %s WHERE v1=0 AND k2=0", true);
+        test("SELECT * FROM %s WHERE v1=0 AND c1=0", true);
+        test("SELECT * FROM %s WHERE v1=0 AND c2=0", true);
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0", true);
+
+        // with token restrictions
+        test("SELECT * FROM %s WHERE v1=0 AND token(k1, k2) = token(0, 0)", false);
+        test("SELECT * FROM %s WHERE v1=0 AND token(k1, k2) > token(0, 0)", false);
+        test("SELECT * FROM %s WHERE v1=0 AND token(k1, k2) > token(0, 0) AND token(k1, k2) <= token(1, 1)", false);
+
+        // with restriction on partition key
+        test("SELECT * FROM %s WHERE v1=0 AND k1=0 AND k2=0", false);
+        test("SELECT * FROM %s WHERE v1=0 AND k1=0 AND k2=0 AND c2=0", true);
+        test("SELECT * FROM %s WHERE v1=0 AND k1=0 AND k2=0 AND v2=0", true);
+
+        // with restriction on partition key and clustering key prefix
+        test("SELECT * FROM %s WHERE v1=0 AND k1=0 AND k2=0 AND c1=0", false);
+        test("SELECT * FROM %s WHERE v1=0 AND k1=0 AND k2=0 AND c1=0 AND v2=0", true);
+
+        // with restriction on partition key and full clustering key
+        test("SELECT * FROM %s WHERE v1=0 AND k1=0 AND k2=0 AND c1=0 AND c2=0", false);
+        test("SELECT * FROM %s WHERE v1=0 AND k1=0 AND k2=0 AND c1=0 AND c2=0 AND v2=0", true);
+
+        // with restriction on partition key and full clustering key, multicolumn format
+        test("SELECT * FROM %s WHERE v1=0 AND k1=0 AND k2=0 AND (c1, c2) = (0, 0)", false);
+        test("SELECT * FROM %s WHERE v1=0 AND k1=0 AND k2=0 AND (c1, c2) = (0, 0) AND v2=0", true);
+    }
+
+    @Test
+    public void testAllowFilteringOnMultipleRegularColumns() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k1 int, k2 int, c1 int, c2 int, v1 int, v2 int, v3 int, " +
+                    "PRIMARY KEY ((k1, k2), c1, c2))");
+        createIndex(format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(format("CREATE CUSTOM INDEX ON %%s(v2) USING '%s'", StorageAttachedIndex.class.getName()));
+        waitForIndexQueryable();
+
+        // with only index restrictions
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0", false);
+        test("SELECT * FROM %s WHERE v1>0 AND v2=0", false);
+        test("SELECT * FROM %s WHERE v1>0 AND v1<1 AND v2=0", false);
+        test("SELECT * FROM %s WHERE v1=0 AND v2>0", false);
+        test("SELECT * FROM %s WHERE v1=0 AND v2>0 AND v2<1", false);
+        test("SELECT * FROM %s WHERE v1>0 AND v1<1 AND v2>0 AND v2<1", false);
+
+        // with additional simple filtering restrictions
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND k1=0", true);
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND k2=0", true);
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND c1=0", true);
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND c2=0", true);
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND v3=0", true);
+
+        // with token restrictions
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND token(k1, k2) = token(0, 0)", false);
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND token(k1, k2) > token(0, 0)", false);
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND token(k1, k2) > token(0, 0) AND token(k1, k2) <= token(1, 1)", false);
+
+        // with restriction on partition key
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND k1=0 AND k2=0", false);
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND k1=0 AND k2=0 AND c2=0", true);
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND k1=0 AND k2=0 AND v3=0", true);
+
+        // with restriction on partition key and clustering key prefix
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND k1=0 AND k2=0 AND c1=0", false);
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND k1=0 AND k2=0 AND c1=0 AND v3=0", true);
+
+        // with restriction on partition key and full clustering key
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND k1=0 AND k2=0 AND c1=0 AND c2=0", false);
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND k1=0 AND k2=0 AND c1=0 AND c2=0 AND v3=0", true);
+
+        // with restriction on partition key and full clustering key, multicolumn format
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND k1=0 AND k2=0 AND (c1, c2) = (0, 0)", false);
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND k1=0 AND k2=0 AND (c1, c2) = (0, 0) AND v3=0", true);
+    }
+
+    @Test
+    public void testAllowFilteringOnClusteringAndRegularColumns() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k1 int, k2 int, c1 int, c2 int, c3 int, c4 int, v1 int, v2 int, v3 int, " +
+                    "PRIMARY KEY ((k1, k2), c1, c2, c3, c4))");
+        createIndex(format("CREATE CUSTOM INDEX ON %%s(c2) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(format("CREATE CUSTOM INDEX ON %%s(c4) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName()));
+        createIndex(format("CREATE CUSTOM INDEX ON %%s(v2) USING '%s'", StorageAttachedIndex.class.getName()));
+        waitForIndexQueryable();
+
+        // with only index restrictions
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0 AND v2=0", false);
+        test("SELECT * FROM %s WHERE c2>0 AND c4>0 AND v1>0 AND v2>0", false);
+        test("SELECT * FROM %s WHERE c2>0 AND c2<1 AND c4>0 AND c4<1 AND v1>0 AND v1<0 AND v2>0 AND v2<1", false);
+
+        // with additional simple filtering restrictions
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0 AND v2=0 AND k1=0", true);
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0 AND v2=0 AND k2=0", true);
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0 AND v2=0 AND c3=0", true);
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0 AND v2=0 AND v3=0", true);
+
+        // with token restrictions
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0 AND v2=0 AND token(k1, k2) = token(0, 0)", false);
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0 AND v2=0 AND token(k1, k2) > token(0, 0)", false);
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0 AND v2=0 AND token(k1, k2) > token(0, 0) AND token(k1, k2) <= token(1, 1)", false);
+
+        // with restriction on partition key
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0 AND v2=0 AND k1=0 AND k2=0", false);
+
+        // with restriction on partition key and clustering key prefix
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0 AND v2=0 AND k1=0 AND k2=0 AND c1=0", true);
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0 AND v2=0 AND k1=0 AND k2=0 AND c1=0 AND v3=0", true);
+
+        // with restriction on partition key and full clustering key
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0 AND v2=0 AND k1=0 AND k2=0 AND c1=0 AND c3=0", false);
+        test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0 AND v2=0 AND k1=0 AND k2=0 AND c1=0 AND c3=0 AND v3=0", true);
+
+        // with restriction on partition key and full clustering key, multicolumn format
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND k1=0 AND k2=0 AND (c1, c2, c3, c4) = (0, 0, 0, 0)", false);
+        test("SELECT * FROM %s WHERE v1=0 AND v2=0 AND k1=0 AND k2=0 AND (c1, c2, c3, c4) = (0, 0, 0, 0) AND v3=0", true);
+    }
+
+    private void test(String query, boolean requiresAllowFiltering) throws Throwable
+    {
+        if (requiresAllowFiltering)
+            assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, query);
+        else
+            assertNotNull(execute(query));
+
+        assertNotNull(execute(query + " ALLOW FILTERING"));
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/BaseDataModel.java b/test/unit/org/apache/cassandra/index/sai/cql/BaseDataModel.java
index 2772dbc0d0..67621c3b9a 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/BaseDataModel.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/BaseDataModel.java
@@ -40,10 +40,9 @@ class BaseDataModel
     public static final String KEYSPACE = "sai_query_keyspace";
 
     public static final String SIMPLE_SELECT_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? LIMIT ?";
-    public static final String SIMPLE_SELECT_WITH_FILTERING_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? LIMIT ? ALLOW FILTERING";
-    public static final String TWO_CLAUSE_AND_QUERY_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? AND %s %s ? LIMIT ? ALLOW FILTERING";
-    public static final String TWO_CLAUSE_AND_QUERY_FILTERING_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? AND %s %s ? LIMIT ? ALLOW FILTERING";
-    public static final String THREE_CLAUSE_AND_QUERY_FILTERING_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? AND %s %s ? AND %s %s ? LIMIT ? ALLOW FILTERING";
+    public static final String RANGE_QUERY_TEMPLATE = "SELECT %s FROM %%s WHERE %s > ? AND %<s < ? LIMIT ?";
+    public static final String TWO_CLAUSE_AND_QUERY_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? AND %s %s ? LIMIT ?";
+    public static final String THREE_CLAUSE_AND_QUERY_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? AND %s %s ? AND %s %s ? LIMIT ?";
 
     public static final String ASCII_COLUMN = "abbreviation";
     public static final String BIGINT_COLUMN = "gdp";
@@ -163,9 +162,9 @@ class BaseDataModel
         String keyColumnDefs = keyColumns.stream().map(column -> column.left + ' ' + column.right).collect(Collectors.joining(", "));
         String normalColumnDefs = columns.stream().map(column -> column.left + ' ' + column.right).collect(Collectors.joining(", "));
 
-        String template = "CREATE TABLE %s (%s, %s, PRIMARY KEY (%s))" + tableOptions;
-        tester.createTable(String.format(template, KEYSPACE + '.' + indexedTable, keyColumnDefs, normalColumnDefs, primaryKey));
-        tester.createTable(String.format(template, KEYSPACE + '.' + nonIndexedTable, keyColumnDefs, normalColumnDefs, primaryKey));
+        String template = "CREATE TABLE %s.%s (%s, %s, PRIMARY KEY (%s))" + tableOptions;
+        tester.createTable(String.format(template, KEYSPACE, indexedTable, keyColumnDefs, normalColumnDefs, primaryKey));
+        tester.createTable(String.format(template, KEYSPACE, nonIndexedTable, keyColumnDefs, normalColumnDefs, primaryKey));
     }
 
     public void createIndexes(Executor tester) throws Throwable
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/CollectionIndexingTest.java b/test/unit/org/apache/cassandra/index/sai/cql/CollectionIndexingTest.java
index be09c193e3..a3906ab6dc 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/CollectionIndexingTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/CollectionIndexingTest.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
 import org.junit.Before;
 import org.junit.Test;
 
+import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
 import org.apache.cassandra.index.sai.SAITester;
 
 import static org.junit.Assert.assertEquals;
@@ -32,10 +33,6 @@ import static org.junit.Assert.assertEquals;
 // is in the cql/types/collections package
 public class CollectionIndexingTest extends SAITester
 {
-    private static final String ALLOW_FILTERING_ERROR = "Cannot execute this query as it might involve data filtering " +
-                                                        "and thus may have unpredictable performance. If you want to " +
-                                                        "execute this query despite the performance unpredictability, " +
-                                                        "use ALLOW FILTERING";
     @Before
     public void setup()
     {
@@ -68,7 +65,7 @@ public class CollectionIndexingTest extends SAITester
     {
         createPopulatedMap("CREATE CUSTOM INDEX ON %s(ENTRIES(value)) USING 'StorageAttachedIndex'");
         assertEquals(2, execute("SELECT * FROM %s WHERE value[1] = 'v1'").size());
-        assertEquals(1, execute("SELECT * FROM %s WHERE value[1] = 'v1' AND value[2] = 'v2' ALLOW FILTERING").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE value[1] = 'v1' AND value[2] = 'v2'").size());
     }
 
     @Test
@@ -93,16 +90,14 @@ public class CollectionIndexingTest extends SAITester
     public void indexFrozenMapQueryKeys() throws Throwable
     {
         createPopulatedFrozenMap("CREATE CUSTOM INDEX ON %s(FULL(value)) USING 'StorageAttachedIndex'");
-        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains key 1");
-        assertEquals(2, execute("SELECT * FROM %s WHERE value contains key 1 ALLOW FILTERING").size());
+        assertUnsupportedIndexOperator(2, "SELECT * FROM %s WHERE value contains key 1");
     }
 
     @Test
     public void indexFrozenMapQueryValues() throws Throwable
     {
         createPopulatedFrozenMap("CREATE CUSTOM INDEX ON %s(FULL(value)) USING 'StorageAttachedIndex'");
-        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains 'v1'");
-        assertEquals(2, execute("SELECT * FROM %s WHERE value contains 'v1' ALLOW FILTERING").size());
+        assertUnsupportedIndexOperator(2, "SELECT * FROM %s WHERE value contains 'v1'");
     }
 
     @Test
@@ -125,16 +120,14 @@ public class CollectionIndexingTest extends SAITester
     public void indexMapEntriesQueryKeys() throws Throwable
     {
         createPopulatedMap("CREATE CUSTOM INDEX ON %s(ENTRIES(value)) USING 'StorageAttachedIndex'");
-        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains key 1");
-        assertEquals(2, execute("SELECT * FROM %s WHERE value contains key 1 ALLOW FILTERING").size());
+        assertUnsupportedIndexOperator(2, "SELECT * FROM %s WHERE value contains key 1");
     }
 
     @Test
     public void indexMapEntriesQueryValues() throws Throwable
     {
         createPopulatedMap("CREATE CUSTOM INDEX ON %s(ENTRIES(value)) USING 'StorageAttachedIndex'");
-        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains 'v1'");
-        assertEquals(2, execute("SELECT * FROM %s WHERE value contains 'v1' ALLOW FILTERING").size());
+        assertUnsupportedIndexOperator(2, "SELECT * FROM %s WHERE value contains 'v1'");
     }
 
     @Test
@@ -149,16 +142,14 @@ public class CollectionIndexingTest extends SAITester
     public void indexMapKeysQueryValues() throws Throwable
     {
         createPopulatedMap("CREATE CUSTOM INDEX ON %s(KEYS(value)) USING 'StorageAttachedIndex'");
-        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains 'v1'");
-        assertEquals(2, execute("SELECT * FROM %s WHERE value contains 'v1' ALLOW FILTERING").size());
+        assertUnsupportedIndexOperator(2, "SELECT * FROM %s WHERE value contains 'v1'");
     }
 
     @Test
     public void indexMapKeysQueryEntries() throws Throwable
     {
         createPopulatedMap("CREATE CUSTOM INDEX ON %s(KEYS(value)) USING 'StorageAttachedIndex'");
-        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value[1] = 'v1'");
-        assertEquals(2, execute("SELECT * FROM %s WHERE value[1] = 'v1' ALLOW FILTERING").size());
+        assertUnsupportedIndexOperator(2, "SELECT * FROM %s WHERE value[1] = 'v1'");
     }
 
     @Test
@@ -173,16 +164,14 @@ public class CollectionIndexingTest extends SAITester
     public void indexMapValuesQueryKeys() throws Throwable
     {
         createPopulatedMap("CREATE CUSTOM INDEX ON %s(VALUES(value)) USING 'StorageAttachedIndex'");
-        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains key 1");
-        assertEquals(2, execute("SELECT * FROM %s WHERE value contains key 1 ALLOW FILTERING").size());
+        assertUnsupportedIndexOperator(2, "SELECT * FROM %s WHERE value contains key 1");
     }
 
     @Test
     public void indexMapValuesQueryEntries() throws Throwable
     {
         createPopulatedMap("CREATE CUSTOM INDEX ON %s(VALUES(value)) USING 'StorageAttachedIndex'");
-        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value[1] = 'v1'");
-        assertEquals(2, execute("SELECT * FROM %s WHERE value[1] = 'v1' ALLOW FILTERING").size());
+        assertUnsupportedIndexOperator(2, "SELECT * FROM %s WHERE value[1] = 'v1'");
     }
 
     private void createPopulatedMap(String createIndex) throws Throwable
@@ -200,6 +189,7 @@ public class CollectionIndexingTest extends SAITester
         }});
     }
 
+    @SuppressWarnings("SameParameterValue")
     private void createPopulatedFrozenMap(String createIndex) throws Throwable
     {
         createTable("CREATE TABLE %s (pk int primary key, value frozen<map<int, text>>)");
@@ -215,6 +205,7 @@ public class CollectionIndexingTest extends SAITester
         }});
     }
 
+    @SuppressWarnings("SameParameterValue")
     private void createPopulatedFrozenList(String createIndex) throws Throwable
     {
         createTable("CREATE TABLE %s (pk int primary key, value frozen<list<int>>)");
@@ -226,8 +217,10 @@ public class CollectionIndexingTest extends SAITester
         execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 4, Arrays.asList(1, 2, 7));
     }
 
-    private void assertUnsupportedIndexOperator(String query, Object... values) throws Throwable
+    @SuppressWarnings("SameParameterValue")
+    private void assertUnsupportedIndexOperator(int expectedSize, String query, Object... values) throws Throwable
     {
-        assertInvalidMessage(ALLOW_FILTERING_ERROR, query, values);
+        assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE, query, values);
+        assertEquals(expectedSize, execute(query + " ALLOW FILTERING").size());
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/CompositePartitionKeyIndexTest.java b/test/unit/org/apache/cassandra/index/sai/cql/CompositePartitionKeyIndexTest.java
index 6bf9891659..bfaabf5b9a 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/CompositePartitionKeyIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/CompositePartitionKeyIndexTest.java
@@ -92,10 +92,10 @@ public class CompositePartitionKeyIndexTest extends SAITester
         assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk2 = '2'"),
                 expectedRow(2));
 
-        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 > 1 AND pk2 = '2' ALLOW FILTERING"),
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 > 1 AND pk2 = '2'"),
                 expectedRow(2));
 
-        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 = -1 AND pk2 = '2' ALLOW FILTERING"));
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 = -1 AND pk2 = '2'"));
 
         assertThatThrownBy(()->execute("SELECT * FROM %s WHERE pk1 = -1 AND val = 2"))
                 .hasMessageContaining("use ALLOW FILTERING");
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/IndexQuerySupport.java b/test/unit/org/apache/cassandra/index/sai/cql/IndexQuerySupport.java
index 8305ac0e50..7573fc96d9 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/IndexQuerySupport.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/IndexQuerySupport.java
@@ -29,7 +29,11 @@ import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 
+import org.junit.Assert;
+
+import com.datastax.driver.core.exceptions.InvalidQueryException;
 import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
 import org.apache.cassandra.db.marshal.InetAddressType;
 import org.apache.cassandra.db.marshal.SimpleDateType;
 import org.apache.cassandra.db.marshal.TimeType;
@@ -37,6 +41,7 @@ import org.apache.cassandra.db.marshal.TimestampType;
 import org.apache.cassandra.db.marshal.UUIDType;
 import org.apache.cassandra.index.sai.plan.StorageAttachedIndexSearcher;
 import org.apache.cassandra.utils.Pair;
+import org.assertj.core.api.Assertions;
 import org.hamcrest.Matchers;
 
 import static org.junit.Assert.assertEquals;
@@ -356,17 +361,19 @@ public class IndexQuerySupport
             andQuery(tester, model,
                      BaseDataModel.TEXT_COLUMN, Operator.EQ, "Alaska",
                      firstPartitionKey, Operator.EQ, 0,
-                     true);
+                     false);
+
+            boolean hasSimplePartitionKey = !(model instanceof BaseDataModel.CompositePartitionKeyDataModel);
 
             andQuery(tester, model,
                      BaseDataModel.TEXT_COLUMN, Operator.EQ, "Kentucky",
                      firstPartitionKey, Operator.GT, 4,
-                     true);
+                     hasSimplePartitionKey);
 
             andQuery(tester, model,
                      BaseDataModel.TEXT_COLUMN, Operator.EQ, "Wyoming",
                      firstPartitionKey, Operator.LT, 200,
-                     true);
+                     hasSimplePartitionKey);
 
             if (model.keyColumns().size() > 1)
             {
@@ -375,40 +382,42 @@ public class IndexQuerySupport
                 andQuery(tester, model,
                          BaseDataModel.BIGINT_COLUMN, Operator.EQ, 4800000000L,
                          secondPrimaryKey, Operator.EQ, 0,
-                         true);
+                         hasSimplePartitionKey);
 
                 andQuery(tester, model,
                          BaseDataModel.DOUBLE_COLUMN, Operator.EQ, 82169.60,
                          secondPrimaryKey, Operator.GT, 0,
-                         true);
+                         hasSimplePartitionKey);
 
                 andQuery(tester, model,
                          BaseDataModel.DOUBLE_COLUMN, Operator.LT, 1948.54,
                          secondPrimaryKey, Operator.LTE, 2,
-                         true);
+                         hasSimplePartitionKey);
 
                 andQuery(tester, model,
                          BaseDataModel.TEXT_COLUMN, Operator.EQ, "Alaska",
                          firstPartitionKey, Operator.EQ, 0,
-                         secondPrimaryKey, Operator.GTE, -1);
+                         secondPrimaryKey, Operator.GTE, -1,
+                         false);
 
                 andQuery(tester, model,
                          BaseDataModel.TEXT_COLUMN, Operator.EQ, "Kentucky",
                          firstPartitionKey, Operator.GT, 4,
-                         secondPrimaryKey, Operator.LT, 50);
+                         secondPrimaryKey, Operator.LT, 50,
+                         hasSimplePartitionKey);
 
                 andQuery(tester, model,
                          BaseDataModel.TEXT_COLUMN, Operator.EQ, "Wyoming",
                          firstPartitionKey, Operator.LT, 200,
-                         secondPrimaryKey, Operator.GT, 0);
+                         secondPrimaryKey, Operator.GT, 0,
+                         hasSimplePartitionKey);
             }
         }
 
         void query(BaseDataModel.Executor tester, BaseDataModel model, String column, Operator operator, Object value)
         {
             String query = String.format(BaseDataModel.SIMPLE_SELECT_TEMPLATE, BaseDataModel.ASCII_COLUMN, column, operator);
-            String queryValidator = String.format(BaseDataModel.SIMPLE_SELECT_WITH_FILTERING_TEMPLATE, BaseDataModel.ASCII_COLUMN, column, operator);
-            validate(tester, model, query, queryValidator, value, limit);
+            validate(tester, model, query, false, value, limit);
         }
 
         void andQuery(BaseDataModel.Executor tester, BaseDataModel model,
@@ -416,55 +425,59 @@ public class IndexQuerySupport
                       String column2, Operator operator2, Object value2,
                       boolean filtering)
         {
-            String query = String.format(filtering ? BaseDataModel.TWO_CLAUSE_AND_QUERY_FILTERING_TEMPLATE : BaseDataModel.TWO_CLAUSE_AND_QUERY_TEMPLATE,
+            String query = String.format(BaseDataModel.TWO_CLAUSE_AND_QUERY_TEMPLATE,
                                          BaseDataModel.ASCII_COLUMN, column1, operator1, column2, operator2);
 
-            String queryValidator = String.format(BaseDataModel.TWO_CLAUSE_AND_QUERY_FILTERING_TEMPLATE,
-                                                  BaseDataModel.ASCII_COLUMN, column1, operator1, column2, operator2);
-
-            validate(tester, model,query, queryValidator, value1, value2, limit);
+            validate(tester, model, query, filtering, value1, value2, limit);
         }
 
         void andQuery(BaseDataModel.Executor tester, BaseDataModel model,
                       String column1, Operator operator1, Object value1,
                       String column2, Operator operator2, Object value2,
-                      String column3, Operator operator3, Object value3)
+                      String column3, Operator operator3, Object value3,
+                      boolean filtering)
         {
-            // TODO: If we support indexes in all columns, ALLOW FILTERING might go away here...
-            String query = String.format(BaseDataModel.THREE_CLAUSE_AND_QUERY_FILTERING_TEMPLATE,
+            String query = String.format(BaseDataModel.THREE_CLAUSE_AND_QUERY_TEMPLATE,
                                          BaseDataModel.ASCII_COLUMN, column1, operator1, column2, operator2, column3, operator3);
 
-            String queryValidator = String.format(BaseDataModel.THREE_CLAUSE_AND_QUERY_FILTERING_TEMPLATE,
-                                                  BaseDataModel.ASCII_COLUMN, column1, operator1, column2, operator2, column3, operator3);
-
-            validate(tester, model, query, queryValidator, value1, value2, value3, limit);
+            validate(tester, model, query, filtering, value1, value2, value3, limit);
         }
 
         void rangeQuery(BaseDataModel.Executor tester, BaseDataModel model, String column, Object value1, Object value2)
         {
-            String template = "SELECT %s FROM %%s WHERE %s > ? AND %s < ? LIMIT ? ALLOW FILTERING";
-            String templateWithFiltering = "SELECT %s FROM %%s WHERE %s > ? AND %s < ? LIMIT ? ALLOW FILTERING";
-
-            String query = String.format(template, BaseDataModel.ASCII_COLUMN, column, column);
-            String queryValidator = String.format(templateWithFiltering, BaseDataModel.ASCII_COLUMN, column, column);
-            validate(tester, model, query, queryValidator, value1, value2, limit);
+            String query = String.format(BaseDataModel.RANGE_QUERY_TEMPLATE, BaseDataModel.ASCII_COLUMN, column);
+            validate(tester, model, query, false, value1, value2, limit);
         }
 
-        private void validate(BaseDataModel.Executor tester, BaseDataModel model, String query, String validator, Object... values)
+        private void validate(BaseDataModel.Executor tester, BaseDataModel model, String query, boolean needsAllowFiltering, Object... values)
         {
             try
             {
                 tester.counterReset();
 
-                List<Object> actual = model.executeIndexed(tester, query, fetchSize, values);
+                // The non indexed query we use to validate the indexed query results is just the very same query but
+                // with ALLOW FILTERING appended. It might happen that the non indexed query also requires ALLOW
+                // FILTERING because it combines indexed and unindexed columns.
+                Assert.assertFalse(query.contains("ALLOW FILTERING"));
+                String validationQuery = query + " ALLOW FILTERING";
+                String indexedQuery = needsAllowFiltering ? validationQuery : query;
+
+                List<Object> actual = model.executeIndexed(tester, indexedQuery, fetchSize, values);
 
                 // This could be more strict, but it serves as a reasonable paging-aware lower bound:
                 int pageCount = (int) Math.ceil(actual.size() / (double) Math.min(actual.size(), fetchSize));
                 assertThat("Expected more calls to " + StorageAttachedIndexSearcher.class, tester.getCounter(), Matchers.greaterThanOrEqualTo((long) Math.max(1, pageCount)));
 
-                List<Object> expected = model.executeNonIndexed(tester, validator, fetchSize, values);
-
+                List<Object> expected = model.executeNonIndexed(tester, validationQuery, fetchSize, values);
                 assertEquals(expected, actual);
+
+                // verify that the query actually requires ALLOW FILTERING
+                if (needsAllowFiltering)
+                {
+                    Assertions.assertThatThrownBy(() -> model.executeIndexed(tester, query, fetchSize, values))
+                              .isInstanceOf(InvalidQueryException.class)
+                              .hasMessageContaining(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE);
+                }
             }
             catch (Throwable ex)
             {
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/MixedIndexImplementationsTest.java b/test/unit/org/apache/cassandra/index/sai/cql/MixedIndexImplementationsTest.java
index 79bb7ac658..f705240d68 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/MixedIndexImplementationsTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/MixedIndexImplementationsTest.java
@@ -74,11 +74,11 @@ public class MixedIndexImplementationsTest extends SAITester
         assertRowsIgnoringOrder(execute(ossSelect, 0), new Object[][]{{0, 0, 0}, {1, 0, 1}});
         assertRowsIgnoringOrder(execute(ossSelect, 1), new Object[][]{{2, 1, 0}, {3, 1, 1}});
 
-        String ndiSelect = "SELECT * FROM %s WHERE v1 = ? AND v2 = ? ALLOW FILTERING";
-        assertRowsIgnoringOrder(execute(ndiSelect, 0, 0), new Object[]{0, 0, 0});
-        assertRowsIgnoringOrder(execute(ndiSelect, 0, 1), new Object[]{1, 0, 1});
-        assertRowsIgnoringOrder(execute(ndiSelect, 1, 0), new Object[]{2, 1, 0});
-        assertRowsIgnoringOrder(execute(ndiSelect, 1, 1), new Object[]{3, 1, 1});
+        String saiSelect = "SELECT * FROM %s WHERE v1 = ? AND v2 = ? ALLOW FILTERING";
+        assertRowsIgnoringOrder(execute(saiSelect, 0, 0), new Object[]{0, 0, 0});
+        assertRowsIgnoringOrder(execute(saiSelect, 0, 1), new Object[]{1, 0, 1});
+        assertRowsIgnoringOrder(execute(saiSelect, 1, 0), new Object[]{2, 1, 0});
+        assertRowsIgnoringOrder(execute(saiSelect, 1, 1), new Object[]{3, 1, 1});
     }
 
     @Test
@@ -107,17 +107,17 @@ public class MixedIndexImplementationsTest extends SAITester
         testAllowFiltering("SELECT * FROM %s WHERE c4=0", false);
         testAllowFiltering("SELECT * FROM %s WHERE r1=0", false);
         testAllowFiltering("SELECT * FROM %s WHERE r2=0", false);
-        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND c3=0", true);
-        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND r1=0", true);
-        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND r2=0", true);
-        testAllowFiltering("SELECT * FROM %s WHERE c3=0 AND r1=0", true);
-        testAllowFiltering("SELECT * FROM %s WHERE c3=0 AND r2=0", true);
-        testAllowFiltering("SELECT * FROM %s WHERE r1=0 AND r2=0", true);
-        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND c3=0 AND r1=0", true);
-        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND c3=0 AND r2=0", true);
-        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND r1=0 AND r2=0", true);
-        testAllowFiltering("SELECT * FROM %s WHERE c3=0 AND r1=0 AND r2=0", true);
-        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND c3=0 AND r1=0 AND r2=0", true);
+        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND c3=0", false);
+        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND r1=0", false);
+        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND r2=0", false);
+        testAllowFiltering("SELECT * FROM %s WHERE c3=0 AND r1=0", false);
+        testAllowFiltering("SELECT * FROM %s WHERE c3=0 AND r2=0", false);
+        testAllowFiltering("SELECT * FROM %s WHERE r1=0 AND r2=0", false);
+        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND c3=0 AND r1=0", false);
+        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND c3=0 AND r2=0", false);
+        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND r1=0 AND r2=0", false);
+        testAllowFiltering("SELECT * FROM %s WHERE c3=0 AND r1=0 AND r2=0", false);
+        testAllowFiltering("SELECT * FROM %s WHERE c2=0 AND c3=0 AND r1=0 AND r2=0", false);
 
         // using the not-SAI index on partition key
         testAllowFiltering("SELECT * FROM %s WHERE k1=0", false);
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/MultipleColumnIndexTest.java b/test/unit/org/apache/cassandra/index/sai/cql/MultipleColumnIndexTest.java
index 9190029588..c29b909c8c 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/MultipleColumnIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/MultipleColumnIndexTest.java
@@ -52,12 +52,12 @@ public class MultipleColumnIndexTest extends SAITester
         execute("INSERT INTO %s(id, text_map) values (2, {'k1':'v1', 'k3':'v3'})");
         execute("INSERT INTO %s(id, text_map) values (3, {'k4':'v4', 'k5':'v5'})");
 
-        assertEquals(1, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map['k2'] = 'v2' ALLOW FILTERING").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map['k2'] = 'v2'").size());
         assertEquals(2, execute("SELECT * FROM %s WHERE text_map CONTAINS 'v1'").size());
         assertEquals(2, execute("SELECT * FROM %s WHERE text_map CONTAINS KEY 'k1'").size());
-        assertEquals(1, execute("SELECT * FROM %s WHERE text_map CONTAINS KEY 'k1' AND text_map CONTAINS KEY 'k2' ALLOW FILTERING").size());
-        assertEquals(2, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map CONTAINS KEY 'k1' AND text_map CONTAINS 'v1' ALLOW FILTERING").size());
-        assertEquals(1, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map CONTAINS KEY 'k1' AND text_map CONTAINS KEY 'k2' AND text_map CONTAINS 'v1' ALLOW FILTERING").size());
-        assertEquals(0, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map CONTAINS KEY 'k1' AND text_map CONTAINS KEY 'k4' ALLOW FILTERING").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE text_map CONTAINS KEY 'k1' AND text_map CONTAINS KEY 'k2'").size());
+        assertEquals(2, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map CONTAINS KEY 'k1' AND text_map CONTAINS 'v1'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map CONTAINS KEY 'k1' AND text_map CONTAINS KEY 'k2' AND text_map CONTAINS 'v1'").size());
+        assertEquals(0, execute("SELECT * FROM %s WHERE text_map['k1'] = 'v1' AND text_map CONTAINS KEY 'k1' AND text_map CONTAINS KEY 'k4'").size());
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/QuerySet.java b/test/unit/org/apache/cassandra/index/sai/cql/types/QuerySet.java
index f491fa073a..d542e42c2b 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/QuerySet.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/QuerySet.java
@@ -147,7 +147,7 @@ public abstract class QuerySet extends SAITester
                 Iterator<?> valueIterator = ((Collection<?>) allRows[index][VALUE_INDEX]).iterator();
                 Object value1 = valueIterator.next();
                 Object value2 = valueIterator.hasNext() ? valueIterator.next() : value1;
-                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value CONTAINS ? AND value CONTAINS ? ALLOW FILTERING",
+                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value CONTAINS ? AND value CONTAINS ?",
                     value1, value2), getExpectedRows(value1, value2, allRows));
             }
         }
@@ -225,7 +225,7 @@ public abstract class QuerySet extends SAITester
                 Map<?, ?> map = (Map<?, ?>)allRows[index][VALUE_INDEX];
                 Object value1 = map.values().toArray()[getRandom().nextIntBetween(0, map.values().size() - 1)];
                 Object value2 = map.keySet().toArray()[getRandom().nextIntBetween(0, map.values().size() - 1)];
-                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value CONTAINS ? AND value CONTAINS ? ALLOW FILTERING",
+                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value CONTAINS ? AND value CONTAINS ?",
                         value1, value2), getExpectedRows(value1, value2, allRows));
             }
         }
@@ -275,7 +275,7 @@ public abstract class QuerySet extends SAITester
                 Map<?, ?> map = (Map<?, ?>)allRows[index][VALUE_INDEX];
                 Object key1 = map.keySet().toArray()[getRandom().nextIntBetween(0, map.keySet().size() - 1)];
                 Object key2 = map.keySet().toArray()[getRandom().nextIntBetween(0, map.keySet().size() - 1)];
-                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value CONTAINS KEY ? AND value CONTAINS KEY ? ALLOW FILTERING",
+                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value CONTAINS KEY ? AND value CONTAINS KEY ?",
                         key1, key2), getExpectedRows(key1, key2, allRows));
             }
         }
@@ -329,7 +329,7 @@ public abstract class QuerySet extends SAITester
                 Object value1 = map.get(key1);
                 Object key2 = map.keySet().toArray()[getRandom().nextIntBetween(0, map.keySet().size() - 1)];
                 Object value2 = map.get(key2);
-                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value[?] = ? AND value[?] = ? ALLOW FILTERING",
+                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value[?] = ? AND value[?] = ?",
                         key1, value1, key2, value2), getExpectedRows(key1, value1, key2, value2, allRows));
             }
         }
@@ -399,16 +399,16 @@ public abstract class QuerySet extends SAITester
                 Object key2 = map.keySet().toArray()[getRandom().nextIntBetween(0, map.keySet().size() - 1)];
                 Object value2 = map.get(key2);
 
-                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value CONTAINS KEY ? AND value CONTAINS KEY ? ALLOW FILTERING", key1, key2),
+                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value CONTAINS KEY ? AND value CONTAINS KEY ?", key1, key2),
                                         getExpectedKeyRows(key1, key2, allRows));
 
-                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value CONTAINS ? AND value CONTAINS ? ALLOW FILTERING", value1, value2),
+                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value CONTAINS ? AND value CONTAINS ?", value1, value2),
                                         getExpectedValueRows(value1, value2, allRows));
 
-                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value[?] = ? AND value[?] = ? ALLOW FILTERING", key1, value1, key2, value2),
+                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value[?] = ? AND value[?] = ?", key1, value1, key2, value2),
                                         getExpectedEntryRows(key1, value1, key2, value2, allRows));
 
-                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value[?] = ? AND value CONTAINS KEY ? AND value CONTAINS ? ALLOW FILTERING", key1, value1, key2, value2),
+                assertRowsIgnoringOrder(tester.execute("SELECT * FROM %s WHERE value[?] = ? AND value CONTAINS KEY ? AND value CONTAINS ?", key1, value1, key2, value2),
                                         getExpectedMixedRows(key1, value1, key2, value2, allRows));
             }
         }
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/SelectiveIntersectionTest.java b/test/unit/org/apache/cassandra/index/sai/disk/SelectiveIntersectionTest.java
index e656e90340..349a24b98c 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/SelectiveIntersectionTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/SelectiveIntersectionTest.java
@@ -92,7 +92,7 @@ public class SelectiveIntersectionTest extends SAITester
     @Test
     public void queryAtSelectiveLimitUsesDeferredFlows() throws Throwable
     {
-        assertEquals(1, execute("SELECT * FROM %s WHERE v1 = '20' AND v2 = '1' ALLOW FILTERING").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE v1 = '20' AND v2 = '1'").size());
         Assert.assertEquals(1, intersectionFlowCounter.get());
 
         Assert.assertEquals(postingsReaderOpenCounter.get(), postingsReaderCloseCounter.get());
@@ -101,7 +101,7 @@ public class SelectiveIntersectionTest extends SAITester
     @Test
     public void queryAboveSelectiveLimitUsesDirectFlows() throws Throwable
     {
-        assertEquals(1, execute("SELECT * FROM %s WHERE v1 = '1' AND v2 = '0' AND v3 = '1' ALLOW FILTERING").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE v1 = '1' AND v2 = '0' AND v3 = '1'").size());
         Assert.assertEquals(1, intersectionFlowCounter.get());
 
         Assert.assertEquals(postingsReaderOpenCounter.get(), postingsReaderCloseCounter.get());
@@ -112,7 +112,7 @@ public class SelectiveIntersectionTest extends SAITester
     {
         setLimits(1);
 
-        assertEquals(1, execute("SELECT * FROM %s WHERE v1 = '1' AND v2 = '0' AND v3 = '1' ALLOW FILTERING").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE v1 = '1' AND v2 = '0' AND v3 = '1'").size());
         Assert.assertEquals(0, intersectionFlowCounter.get());
 
         Assert.assertEquals(postingsReaderOpenCounter.get(), postingsReaderCloseCounter.get());
@@ -123,7 +123,7 @@ public class SelectiveIntersectionTest extends SAITester
     {
         setLimits(0);
 
-        assertEquals(1, execute("SELECT * FROM %s WHERE v1 = '1' AND v2 = '0' AND v3 = '1' ALLOW FILTERING").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE v1 = '1' AND v2 = '0' AND v3 = '1'").size());
         Assert.assertEquals(1, intersectionFlowCounter.get());
 
         Assert.assertEquals(postingsReaderOpenCounter.get(), postingsReaderCloseCounter.get());
@@ -134,13 +134,13 @@ public class SelectiveIntersectionTest extends SAITester
     {
         Session session = sessionNet();
 
-        String trace = getSingleTraceStatement(session, "SELECT * FROM %s WHERE v1 = '1' AND v2 = '0' AND v3 = '1' ALLOW FILTERING", "Selecting");
+        String trace = getSingleTraceStatement(session, "SELECT * FROM %s WHERE v1 = '1' AND v2 = '0' AND v3 = '1'", "Selecting");
 
         assertEquals("Selecting 2 indexes with cardinalities of 1, 10 out of 3 indexes", trace);
 
         setLimits(1);
 
-        trace = getSingleTraceStatement(session, "SELECT * FROM %s WHERE v1 = '1' AND v2 = '0' AND v3 = '1' ALLOW FILTERING", "Selecting");
+        trace = getSingleTraceStatement(session, "SELECT * FROM %s WHERE v1 = '1' AND v2 = '0' AND v3 = '1'", "Selecting");
 
         assertEquals("Selecting 1 index with cardinality of 1 out of 3 indexes", trace);
 
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/SingleNodeQueryFailureTest.java b/test/unit/org/apache/cassandra/index/sai/disk/SingleNodeQueryFailureTest.java
index e417f7ac59..a208aa9770 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/SingleNodeQueryFailureTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/SingleNodeQueryFailureTest.java
@@ -86,13 +86,13 @@ public class SingleNodeQueryFailureTest extends SAITester
         {
             Injections.inject(injection);
 
-            assertThatThrownBy(() -> executeNet("SELECT id FROM %s WHERE v1 = '0' ALLOW FILTERING"))
+            assertThatThrownBy(() -> executeNet("SELECT id FROM %s WHERE v1 = '0'"))
                     .isInstanceOf(ReadFailureException.class);
 
-            assertThatThrownBy(() -> executeNet("SELECT id FROM %s WHERE v1 = '1' ALLOW FILTERING"))
+            assertThatThrownBy(() -> executeNet("SELECT id FROM %s WHERE v1 = '1'"))
                     .isInstanceOf(ReadFailureException.class);
 
-            assertThatThrownBy(() -> executeNet("SELECT id FROM %s WHERE v1 = '2' ALLOW FILTERING"))
+            assertThatThrownBy(() -> executeNet("SELECT id FROM %s WHERE v1 = '2'"))
                     .isInstanceOf(ReadFailureException.class);
         }
         catch (Exception e)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


[cassandra] 05/06: Unify CQLTester#waitForIndex and SAITester#waitForIndexQueryable

Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

maedhroz pushed a commit to branch cep-7-sai
in repository https://gitbox.apache.org/repos/asf/cassandra.git

commit 26f2c942ab475ff80ac9506cfd1408909eb6d8f4
Author: Andrés de la Peña <a....@gmail.com>
AuthorDate: Thu May 11 15:01:00 2023 +0100

    Unify CQLTester#waitForIndex and SAITester#waitForIndexQueryable
    
    patch by Andrés de la Peña; reviewed by Caleb Rackliffe for CASSANDRA-18521
---
 .../cassandra/cql3/functions/types/ParseUtils.java |   4 +-
 test/unit/org/apache/cassandra/cql3/CQLTester.java | 253 +++++++++++++++------
 .../org/apache/cassandra/cql3/CQLTesterTest.java   |  82 +++++++
 .../org/apache/cassandra/cql3/KeyCacheCqlTest.java |   4 +-
 .../entities/SecondaryIndexOnMapEntriesTest.java   |   2 +-
 .../validation/entities/SecondaryIndexTest.java    |  38 ++--
 .../operations/InsertUpdateIfConditionTest.java    |   2 -
 .../db/guardrails/GuardrailTablesTest.java         |   2 +-
 .../cassandra/index/SecondaryIndexManagerTest.java | 127 +++++------
 .../index/internal/CassandraIndexTest.java         |   9 +-
 .../org/apache/cassandra/index/sai/SAITester.java  |  41 +---
 .../index/sai/cql/AllowFilteringTest.java          |   6 -
 .../cassandra/index/sai/cql/BaseDataModel.java     |  29 ++-
 .../cassandra/index/sai/cql/BooleanTypeTest.java   |   1 -
 .../index/sai/cql/CollectionIndexingTest.java      |   3 -
 .../index/sai/cql/DecimalLargeValueTest.java       |   2 -
 .../index/sai/cql/DuplicateRowIDTest.java          |   1 -
 .../sai/cql/MixedIndexImplementationsTest.java     |   4 -
 .../index/sai/cql/MultipleColumnIndexTest.java     |   2 -
 .../index/sai/cql/SingleNodeExecutor.java          |   4 +-
 .../index/sai/cql/StorageAttachedIndexDDLTest.java |  36 ++-
 .../index/sai/cql/types/IndexingTypeSupport.java   |   2 -
 .../cassandra/index/sai/disk/NodeStartupTest.java  |   5 +-
 .../index/sai/disk/SelectiveIntersectionTest.java  |   2 -
 .../index/sai/disk/SingleNodeQueryFailureTest.java |   1 -
 .../index/sai/functional/CompactionTest.java       |   2 -
 .../index/sai/functional/DiskSpaceTest.java        |   1 -
 .../index/sai/functional/DropTableTest.java        |   1 -
 .../index/sai/functional/FailureTest.java          |   2 +-
 .../index/sai/functional/GroupComponentsTest.java  |   3 -
 .../index/sai/functional/NodeRestartTest.java      |   9 +-
 .../index/sai/functional/SnapshotTest.java         |   2 -
 .../index/sai/metrics/QueryMetricsTest.java        |   1 -
 .../sai/metrics/SegmentFlushingFailureTester.java  |   1 -
 .../index/sai/metrics/StateMetricsTest.java        |   6 +-
 .../cassandra/index/sai/plan/OperationTest.java    |   2 +-
 .../index/sai/virtual/IndexesSystemViewTest.java   |   4 +-
 .../index/sai/virtual/SSTablesSystemViewTest.java  |   2 -
 .../index/sai/virtual/SegmentsSystemViewTest.java  |   1 -
 39 files changed, 390 insertions(+), 309 deletions(-)

diff --git a/src/java/org/apache/cassandra/cql3/functions/types/ParseUtils.java b/src/java/org/apache/cassandra/cql3/functions/types/ParseUtils.java
index 8972beed15..8d0f29b45b 100644
--- a/src/java/org/apache/cassandra/cql3/functions/types/ParseUtils.java
+++ b/src/java/org/apache/cassandra/cql3/functions/types/ParseUtils.java
@@ -292,7 +292,7 @@ public abstract class ParseUtils
      * @param value The string to un-double quote.
      * @return The un-double quoted string.
      */
-    static String unDoubleQuote(String value)
+    public static String unDoubleQuote(String value)
     {
         return unquote(value, '"');
     }
@@ -482,7 +482,7 @@ public abstract class ParseUtils
      * @return {@code true} if the given string is surrounded by the quote character, and {@code
      * false} otherwise.
      */
-    private static boolean isQuoted(String value, char quoteChar)
+    public static boolean isQuoted(String value, char quoteChar)
     {
         return value != null
                && value.length() > 1
diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java
index a898faeb36..dc4a0e413e 100644
--- a/test/unit/org/apache/cassandra/cql3/CQLTester.java
+++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java
@@ -48,6 +48,8 @@ import java.util.function.Consumer;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+
+import javax.annotation.Nullable;
 import javax.management.MBeanServerConnection;
 import javax.management.remote.JMXConnector;
 import javax.management.remote.JMXConnectorFactory;
@@ -59,6 +61,8 @@ import com.google.common.base.Objects;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
+import org.assertj.core.api.Assertions;
+import org.awaitility.Awaitility;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -96,6 +100,7 @@ import org.apache.cassandra.config.DataStorageSpec;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.EncryptionOptions;
 import org.apache.cassandra.cql3.functions.FunctionName;
+import org.apache.cassandra.cql3.functions.types.ParseUtils;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.Keyspace;
@@ -128,6 +133,7 @@ import org.apache.cassandra.dht.Murmur3Partitioner;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.exceptions.SyntaxException;
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.SecondaryIndexManager;
 import org.apache.cassandra.io.filesystem.ListenableFileSystem;
 import org.apache.cassandra.io.util.File;
@@ -158,20 +164,17 @@ import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.JMXServerUtils;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.TimeUUID;
-import org.assertj.core.api.Assertions;
-import org.awaitility.Awaitility;
 
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_JMX_LOCAL_PORT;
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_DRIVER_CONNECTION_TIMEOUT_MS;
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_DRIVER_READ_TIMEOUT_MS;
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_REUSE_PREPARED;
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_ROW_CACHE_SIZE;
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_USE_PREPARED;
-import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 /**
  * Base class for CQL tests.
@@ -195,6 +198,7 @@ public abstract class CQLTester
     public static final String DATA_CENTER = ServerTestUtils.DATA_CENTER;
     public static final String DATA_CENTER_REMOTE = ServerTestUtils.DATA_CENTER_REMOTE;
     public static final String RACK1 = ServerTestUtils.RACK1;
+    protected static final int ASSERTION_TIMEOUT_SECONDS = 15;
 
     private static org.apache.cassandra.transport.Server server;
     private static JMXConnectorServer jmxServer;
@@ -1096,122 +1100,229 @@ public abstract class CQLTester
 
     protected void dropTable(String query)
     {
-        dropFormattedTable(String.format(query, KEYSPACE + "." + currentTable()));
+        dropTable(KEYSPACE, query);
+    }
+
+    protected void dropTable(String keyspace, String query)
+    {
+        dropFormattedTable(String.format(query, keyspace + "." + currentTable()));
     }
 
-    protected void dropFormattedTable(String formattedQuery)
+    private void dropFormattedTable(String formattedQuery)
     {
         logger.info(formattedQuery);
         schemaChange(formattedQuery);
     }
 
+    /**
+     * Creates a secondary index, waiting for it to become queryable.
+     *
+     * @param query the index creation query
+     * @return the name of the created index
+     */
     protected String createIndex(String query)
     {
         return createIndex(KEYSPACE, query);
     }
 
+    /**
+     * Creates a secondary index, waiting for it to become queryable.
+     *
+     * @param keyspace the keyspace the created index should belong to
+     * @param query the index creation query
+     * @return the name of the created index
+     */
     protected String createIndex(String keyspace, String query)
     {
         String formattedQuery = formatQuery(keyspace, query);
-        return createFormattedIndex(formattedQuery);
+        Pair<String, String> qualifiedIndexName = createFormattedIndex(keyspace, formattedQuery);
+        waitForIndexQueryable(qualifiedIndexName.left, qualifiedIndexName.right);
+        return qualifiedIndexName.right;
+    }
+
+    /**
+     * Creates a secondary index without waiting for it to become queryable.
+     *
+     * @param query the index creation query
+     * @return the name of the created index
+     */
+    protected String createIndexAsync(String query)
+    {
+        return createIndexAsync(KEYSPACE, query);
+    }
+
+    /**
+     * Creates a secondary index without waiting for it to become queryable.
+     *
+     * @param keyspace the keyspace the created index should belong to
+     * @param query the index creation query
+     * @return the name of the created index
+     */
+    protected String createIndexAsync(String keyspace, String query)
+    {
+        String formattedQuery = formatQuery(keyspace, query);
+        return createFormattedIndex(keyspace, formattedQuery).right;
     }
 
-    protected String createFormattedIndex(String formattedQuery)
+    private Pair<String, String> createFormattedIndex(String keyspace, String formattedQuery)
     {
         logger.info(formattedQuery);
-        String indexName = getCreateIndexName(formattedQuery);
+        Pair<String, String> qualifiedIndexName = getCreateIndexName(keyspace, formattedQuery);
         schemaChange(formattedQuery);
-        return indexName;
+        return qualifiedIndexName;
     }
 
-    protected static String getCreateIndexName(String formattedQuery)
+    protected static Pair<String, String> getCreateIndexName(String keyspace, String formattedQuery)
     {
         Matcher matcher = CREATE_INDEX_PATTERN.matcher(formattedQuery);
         if (!matcher.find())
             throw new IllegalArgumentException("Expected valid create index query but found: " + formattedQuery);
 
+        String parsedKeyspace = matcher.group(5);
+        if (!Strings.isNullOrEmpty(parsedKeyspace))
+            keyspace = parsedKeyspace;
+
         String index = matcher.group(2);
-        if (!Strings.isNullOrEmpty(index))
-            return index;
+        if (Strings.isNullOrEmpty(index))
+        {
+            String table = matcher.group(7);
+            if (Strings.isNullOrEmpty(table))
+                throw new IllegalArgumentException("Table name should be specified: " + formattedQuery);
+
+            String column = matcher.group(9);
+
+            String baseName = Strings.isNullOrEmpty(column)
+                              ? IndexMetadata.generateDefaultIndexName(table)
+                              : IndexMetadata.generateDefaultIndexName(table, new ColumnIdentifier(column, true));
 
-        String keyspace = matcher.group(5);
-        if (Strings.isNullOrEmpty(keyspace))
-            throw new IllegalArgumentException("Keyspace name should be specified: " + formattedQuery);
+            KeyspaceMetadata ks = Schema.instance.getKeyspaceMetadata(keyspace);
+            assertNotNull(ks);
+            index = ks.findAvailableIndexName(baseName);
+        }
 
-        String table = matcher.group(7);
-        if (Strings.isNullOrEmpty(table))
-            throw new IllegalArgumentException("Table name should be specified: " + formattedQuery);
+        index = ParseUtils.isQuoted(index, '\"')
+                ? ParseUtils.unDoubleQuote(index)
+                : index.toLowerCase();
 
-        String column = matcher.group(9);
+        return Pair.create(keyspace, index);
+    }
 
-        String baseName = Strings.isNullOrEmpty(column)
-                        ? IndexMetadata.generateDefaultIndexName(table)
-                        : IndexMetadata.generateDefaultIndexName(table, new ColumnIdentifier(column, true));
+    public void waitForTableIndexesQueryable()
+    {
+        waitForTableIndexesQueryable(currentTable());
+    }
 
-        KeyspaceMetadata ks = Schema.instance.getKeyspaceMetadata(keyspace);
-        return ks.findAvailableIndexName(baseName);
+    public void waitForTableIndexesQueryable(String table)
+    {
+        waitForTableIndexesQueryable(KEYSPACE, table);
     }
 
     /**
-     * Index creation is asynchronous, this method searches in the system table IndexInfo
-     * for the specified index and returns true if it finds it, which indicates the
-     * index was built. If we haven't found it after 5 seconds we give-up.
+     * Index creation is asynchronous. This method waits until all the indexes in the specified table are queryable.
+     *
+     * @param keyspace the table keyspace name
+     * @param table the table name
      */
-    protected boolean waitForIndex(String keyspace, String table, String index) throws Throwable
+    public void waitForTableIndexesQueryable(String keyspace, String table)
     {
-        long start = currentTimeMillis();
-        boolean indexCreated = false;
-        while (!indexCreated)
-        {
-            Object[][] results = getRows(execute("select index_name from system.\"IndexInfo\" where table_name = ?", keyspace));
-            for(int i = 0; i < results.length; i++)
-            {
-                if (index.equals(results[i][0]))
-                {
-                    indexCreated = true;
-                    break;
-                }
-            }
+        waitForAssert(() -> Assertions.assertThat(getNotQueryableIndexes(keyspace, table)).isEmpty(), 60, TimeUnit.SECONDS);
+    }
 
-            if (currentTimeMillis() - start > 5000)
-                break;
+    public void waitForIndexQueryable(String index)
+    {
+        waitForIndexQueryable(KEYSPACE, index);
+    }
 
-            Thread.sleep(10);
-        }
+    /**
+     * Index creation is asynchronous. This method waits until the specified index is queryable.
+     *
+     * @param keyspace the index keyspace name
+     * @param index the index name
+     */
+    public void waitForIndexQueryable(String keyspace, String index)
+    {
+        waitForAssert(() -> assertTrue(isIndexQueryable(keyspace, index)), 60, TimeUnit.SECONDS);
+    }
 
-        return indexCreated;
+    protected void waitForIndexBuilds(String index)
+    {
+        waitForIndexBuilds(KEYSPACE, index);
     }
 
     /**
-     * Index creation is asynchronous, this method waits until the specified index hasn't any building task running.
+     * Index creation is asynchronous. This method waits until the specified index hasn't any building task running.
      * <p>
-     * This method differs from {@link #waitForIndex(String, String, String)} in that it doesn't require the index to be
-     * fully nor successfully built, so it can be used to wait for failing index builds.
+     * This method differs from {@link #waitForIndexQueryable(String, String)} in that it doesn't require the
+     * index to be fully nor successfully built, so it can be used to wait for failing index builds.
      *
      * @param keyspace the index keyspace name
-     * @param indexName the index name
-     * @return {@code true} if the index build tasks have finished in 5 seconds, {@code false} otherwise
+     * @param index the index name
+     */
+    protected void waitForIndexBuilds(String keyspace, String index)
+    {
+        waitForAssert(() -> assertFalse(isIndexBuilding(keyspace, index)), 60, TimeUnit.SECONDS);
+    }
+
+    /**
+     * @return the names of the indexes in the current table that are not queryable
      */
-    protected boolean waitForIndexBuilds(String keyspace, String indexName) throws InterruptedException
+    protected Set<String> getNotQueryableIndexes()
+    {
+        return getNotQueryableIndexes(KEYSPACE, currentTable());
+    }
+
+    /**
+     * @param keyspace the table keyspace name
+     * @param table the table name
+     * @return the names of the indexes in the specified table that are not queryable
+     */
+    protected Set<String> getNotQueryableIndexes(String keyspace, String table)
+    {
+        SecondaryIndexManager sim = Keyspace.open(keyspace).getColumnFamilyStore(table).indexManager;
+        return sim.listIndexes()
+                  .stream()
+                  .filter(index -> !sim.isIndexQueryable(index))
+                  .map(index -> index.getIndexMetadata().name)
+                  .collect(Collectors.toSet());
+    }
+
+    protected boolean isIndexBuilding(String keyspace, String indexName)
+    {
+        SecondaryIndexManager manager = getIndexManager(keyspace, indexName);
+        assertNotNull(manager);
+
+        return manager.isIndexBuilding(indexName);
+    }
+
+    protected boolean isIndexQueryable(String keyspace, String indexName)
     {
-        long start = currentTimeMillis();
-        SecondaryIndexManager indexManager = getCurrentColumnFamilyStore(keyspace).indexManager;
+        SecondaryIndexManager manager = getIndexManager(keyspace, indexName);
+        assertNotNull(manager);
 
-        while (true)
+        Index index = manager.getIndexByName(indexName);
+        return manager.isIndexQueryable(index);
+    }
+
+    @Nullable
+    protected SecondaryIndexManager getIndexManager(String keyspace, String indexName)
+    {
+        for (ColumnFamilyStore cfs : Keyspace.open(keyspace).getColumnFamilyStores())
         {
-            if (!indexManager.isIndexBuilding(indexName))
-            {
-                return true;
-            }
-            else if (currentTimeMillis() - start > 5000)
-            {
-                return false;
-            }
-            else
-            {
-                Thread.sleep(10);
-            }
+            Index index = cfs.indexManager.getIndexByName(indexName);
+            if (index != null)
+                return cfs.indexManager;
         }
+        return null;
+    }
+
+    protected void waitForAssert(Runnable runnableAssert, long timeout, TimeUnit unit)
+    {
+        Awaitility.await().dontCatchUncaughtExceptions().atMost(timeout, unit).untilAsserted(runnableAssert::run);
+    }
+
+    protected void waitForAssert(Runnable assertion)
+    {
+        waitForAssert(assertion, ASSERTION_TIMEOUT_SECONDS, TimeUnit.SECONDS);
     }
 
     protected void createIndexMayThrow(String query) throws Throwable
diff --git a/test/unit/org/apache/cassandra/cql3/CQLTesterTest.java b/test/unit/org/apache/cassandra/cql3/CQLTesterTest.java
new file mode 100644
index 0000000000..052ced34da
--- /dev/null
+++ b/test/unit/org/apache/cassandra/cql3/CQLTesterTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.cql3;
+
+import org.junit.Test;
+
+import org.apache.cassandra.utils.Pair;
+import org.assertj.core.api.Assertions;
+
+public class CQLTesterTest extends CQLTester
+{
+    @Test
+    public void testGetCreateIndexName()
+    {
+        schemaChange("CREATE KEYSPACE k WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}");
+        schemaChange("CREATE KEYSPACE k2 WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}");
+        String table = createTable("CREATE TABLE %s (k int PRIMARY KEY, c int)");
+
+        // standard index with explicit name
+        testGetCreateIndexName("k", "idx", "CREATE INDEX idx ON k.t (c)");
+        testGetCreateIndexName(KEYSPACE, "idx", "CREATE INDEX idx ON t (c)");
+        testGetCreateIndexName(KEYSPACE, "idx", "CREATE INDEX idx ON %s (c)");
+
+        // custom index with explicit name
+        testGetCreateIndexName("k", "idx", "CREATE CUSTOM INDEX idx ON k.t (c) USING 'class'");
+        testGetCreateIndexName(KEYSPACE, "idx", "CREATE CUSTOM INDEX idx ON t (c) USING 'class'");
+        testGetCreateIndexName(KEYSPACE, "idx", "CREATE CUSTOM INDEX idx ON %s (c) USING 'class'");
+
+        // with an unqouted case-sensitive index name
+        testGetCreateIndexName("k", "idx", "CREATE INDEX iDx ON k.t (c)");
+        testGetCreateIndexName(KEYSPACE, "idx", "CREATE INDEX iDx ON t (c)");
+        testGetCreateIndexName(KEYSPACE, "idx", "CREATE INDEX iDx ON %s (c)");
+
+        // with a quoted case-sensitive index name
+        testGetCreateIndexName("k", "iDx", "CREATE INDEX \"iDx\" ON k.t (c)");
+        testGetCreateIndexName(KEYSPACE, "iDx", "CREATE INDEX \"iDx\" ON t (c)");
+        testGetCreateIndexName(KEYSPACE, "iDx", "CREATE INDEX \"iDx\" ON %s (c)");
+
+        // standard index with autogenerated name
+        testGetCreateIndexName("k", "t_c_idx", "CREATE INDEX ON k.t (c)");
+        testGetCreateIndexName(KEYSPACE, "t_c_idx", "CREATE INDEX ON t (c)");
+        testGetCreateIndexName(KEYSPACE, table + "_c_idx", "CREATE INDEX ON %s (c)");
+
+        // custom index with autogenerated name
+        testGetCreateIndexName("k", "t_c_idx", "CREATE CUSTOM INDEX ON k.t (c) USING 'class'");
+        testGetCreateIndexName(KEYSPACE, "t_c_idx", "CREATE CUSTOM INDEX ON t (c) USING 'class'");
+        testGetCreateIndexName(KEYSPACE, table + "_c_idx", "CREATE CUSTOM INDEX ON %s (c) USING 'class'");
+
+        // with a not-default working keyspace
+        testGetCreateIndexName("k", "t_c_idx", "k2", "CREATE INDEX ON k.t (c)");
+        testGetCreateIndexName("k2", "t_c_idx", "k2", "CREATE INDEX ON t (c)");
+        testGetCreateIndexName("k2", table + "_c_idx", "k2", "CREATE INDEX ON %s (c)");
+    }
+
+    private void testGetCreateIndexName(String expectedKeyspaceName, String expectedIndexName, String query)
+    {
+        testGetCreateIndexName(expectedKeyspaceName, expectedIndexName, KEYSPACE, query);
+    }
+
+    private void testGetCreateIndexName(String expectedKeyspaceName, String expectedIndexName, String keyspace, String query)
+    {
+        Pair<String, String> name = getCreateIndexName(keyspace, formatQuery(keyspace, query));
+        Assertions.assertThat(name.left).isEqualTo(expectedKeyspaceName);
+        Assertions.assertThat(name.right).isEqualTo(expectedIndexName);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java b/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java
index 9c7dac159c..3f5f75abd8 100644
--- a/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java
+++ b/test/unit/org/apache/cassandra/cql3/KeyCacheCqlTest.java
@@ -127,13 +127,13 @@ public class KeyCacheCqlTest extends CQLTester
     @Override
     protected String createIndex(String query)
     {
-        return createFormattedIndex(formatQuery(KEYSPACE_PER_TEST, query));
+        return createIndex(KEYSPACE_PER_TEST, query);
     }
 
     @Override
     protected void dropTable(String query)
     {
-        dropFormattedTable(String.format(query, KEYSPACE_PER_TEST + "." + currentTable()));
+        dropTable(KEYSPACE_PER_TEST, query);
     }
 
     @Test
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexOnMapEntriesTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexOnMapEntriesTest.java
index b69948fa07..53d752e3ae 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexOnMapEntriesTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexOnMapEntriesTest.java
@@ -256,7 +256,7 @@ public class SecondaryIndexOnMapEntriesTest extends CQLTester
         assertRows(execute(query, params));
     }
 
-    private void createSimpleTableAndIndex() throws Throwable
+    private void createSimpleTableAndIndex()
     {
         createTable("CREATE TABLE %s (k TEXT PRIMARY KEY, v MAP<TEXT, INT>)");
         createIndex("CREATE INDEX ON %s(ENTRIES(v))");
diff --git a/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java b/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
index 77c611be1a..bf780e000d 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/entities/SecondaryIndexTest.java
@@ -107,8 +107,8 @@ public class SecondaryIndexTest extends CQLTester
                              format("DROP INDEX %s.%s", KEYSPACE, indexName));
 
         createTable("CREATE TABLE %s (a int primary key, b int);");
-        createIndex("CREATE INDEX " + indexName + " ON %s(b);");
-        createIndex("CREATE INDEX IF NOT EXISTS " + indexName + " ON %s(b);");
+        createIndexAsync("CREATE INDEX " + indexName + " ON %s(b);");
+        createIndexAsync("CREATE INDEX IF NOT EXISTS " + indexName + " ON %s(b);");
 
         assertInvalidMessage(format("Index '%s' already exists",
                                     removeQuotes(indexName.toLowerCase(Locale.US))),
@@ -117,7 +117,7 @@ public class SecondaryIndexTest extends CQLTester
         // IF NOT EXISTS should apply in cases where the new index differs from an existing one in name only
         String otherIndexName = "index_" + nanoTime();
         assertEquals(1, getCurrentColumnFamilyStore().metadata().indexes.size());
-        createIndex("CREATE INDEX IF NOT EXISTS " + otherIndexName + " ON %s(b)");
+        createIndexAsync("CREATE INDEX IF NOT EXISTS " + otherIndexName + " ON %s(b)");
         assertEquals(1, getCurrentColumnFamilyStore().metadata().indexes.size());
         assertInvalidMessage(format("Index %s is a duplicate of existing index %s",
                                     removeQuotes(otherIndexName.toLowerCase(Locale.US)),
@@ -254,7 +254,7 @@ public class SecondaryIndexTest extends CQLTester
     @Test
     public void testIndexOnComposite() throws Throwable
     {
-        String tableName = createTable("CREATE TABLE %s (blog_id int, timestamp int, author text, content text, PRIMARY KEY (blog_id, timestamp))");
+        createTable("CREATE TABLE %s (blog_id int, timestamp int, author text, content text, PRIMARY KEY (blog_id, timestamp))");
 
         execute("INSERT INTO %s (blog_id, timestamp, author, content) VALUES (?, ?, ?, ?)", 0, 0, "bob", "1st post");
         execute("INSERT INTO %s (blog_id, timestamp, author, content) VALUES (?, ?, ?, ?)", 0, 1, "tom", "2nd post");
@@ -264,8 +264,6 @@ public class SecondaryIndexTest extends CQLTester
 
         createIndex("CREATE INDEX authoridx ON %s (author)");
 
-        assertTrue(waitForIndex(keyspace(), tableName, "authoridx"));
-
         beforeAndAfterFlush(() -> {
             assertRows(execute("SELECT blog_id, timestamp FROM %s WHERE author = 'bob'"),
                        row(1, 0),
@@ -618,14 +616,13 @@ public class SecondaryIndexTest extends CQLTester
     @Test
     public void testCanQuerySecondaryIndex() throws Throwable
     {
-        String tableName = createTable("CREATE TABLE %s (k int PRIMARY KEY, v int,)");
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v int,)");
 
         execute("ALTER TABLE %s WITH CACHING = { 'keys': 'ALL', 'rows_per_partition': 'ALL' }");
         execute("INSERT INTO %s (k,v) VALUES (0,0)");
         execute("INSERT INTO %s (k,v) VALUES (1,1)");
 
         createIndex("CREATE INDEX testindex on %s (v)");
-        assertTrue(waitForIndex(keyspace(), tableName, "testindex"));
 
         assertRows(execute("SELECT k FROM %s WHERE v = 0"), row(0));
         cleanupCache();
@@ -1040,7 +1037,7 @@ public class SecondaryIndexTest extends CQLTester
             for (int j = 0; j < 10; j++)
                 execute("INSERT INTO %s (pk, ck, value) VALUES (?, ?, ?)", i, j, i + j);
 
-        createIndex("CREATE CUSTOM INDEX testIndex ON %s (value) USING '" + IndexBlockingOnInitialization.class.getName() + "'");
+        createIndexAsync("CREATE CUSTOM INDEX testIndex ON %s (value) USING '" + IndexBlockingOnInitialization.class.getName() + "'");
         try
         {
             execute("SELECT value FROM %s WHERE value = 2");
@@ -1060,9 +1057,8 @@ public class SecondaryIndexTest extends CQLTester
     public void testReadOnlyIndex() throws Throwable
     {
         // On successful initialization both reads and writes go through
-        String tableName = createTable("CREATE TABLE %s (pk int, ck int, value int, PRIMARY KEY (pk, ck))");
+        createTable("CREATE TABLE %s (pk int, ck int, value int, PRIMARY KEY (pk, ck))");
         String indexName = createIndex("CREATE CUSTOM INDEX ON %s (value) USING '" + ReadOnlyOnFailureIndex.class.getName() + "'");
-        assertTrue(waitForIndex(keyspace(), tableName, indexName));
         execute("SELECT value FROM %s WHERE value = 1");
         execute("INSERT INTO %s (pk, ck, value) VALUES (?, ?, ?)", 1, 1, 1);
         ReadOnlyOnFailureIndex index = (ReadOnlyOnFailureIndex) getCurrentColumnFamilyStore().indexManager.getIndexByName(indexName);
@@ -1078,9 +1074,9 @@ public class SecondaryIndexTest extends CQLTester
 
         // On bad initial build writes are not forwarded to the index
         ReadOnlyOnFailureIndex.failInit = true;
-        indexName = createIndex("CREATE CUSTOM INDEX ON %s (value) USING '" + ReadOnlyOnFailureIndex.class.getName() + "'");
+        indexName = createIndexAsync("CREATE CUSTOM INDEX ON %s (value) USING '" + ReadOnlyOnFailureIndex.class.getName() + "'");
         index = (ReadOnlyOnFailureIndex) getCurrentColumnFamilyStore().indexManager.getIndexByName(indexName);
-        assertTrue(waitForIndexBuilds(keyspace(), indexName));
+        waitForIndexBuilds(indexName);
         assertInvalidThrow(IndexNotAvailableException.class, "SELECT value FROM %s WHERE value = 1");
         execute("INSERT INTO %s (pk, ck, value) VALUES (?, ?, ?)", 1, 1, 1);
         assertEquals(0, index.rowsInserted.size());
@@ -1099,9 +1095,8 @@ public class SecondaryIndexTest extends CQLTester
     public void testWriteOnlyIndex() throws Throwable
     {
         // On successful initialization both reads and writes go through
-        String tableName = createTable("CREATE TABLE %s (pk int, ck int, value int, PRIMARY KEY (pk, ck))");
+        createTable("CREATE TABLE %s (pk int, ck int, value int, PRIMARY KEY (pk, ck))");
         String indexName = createIndex("CREATE CUSTOM INDEX ON %s (value) USING '" + WriteOnlyOnFailureIndex.class.getName() + "'");
-        assertTrue(waitForIndex(keyspace(), tableName, indexName));
         execute("SELECT value FROM %s WHERE value = 1");
         execute("INSERT INTO %s (pk, ck, value) VALUES (?, ?, ?)", 1, 1, 1);
         WriteOnlyOnFailureIndex index = (WriteOnlyOnFailureIndex) getCurrentColumnFamilyStore().indexManager.getIndexByName(indexName);
@@ -1117,9 +1112,9 @@ public class SecondaryIndexTest extends CQLTester
 
         // On bad initial build writes are forwarded to the index
         WriteOnlyOnFailureIndex.failInit = true;
-        indexName = createIndex("CREATE CUSTOM INDEX ON %s (value) USING '" + WriteOnlyOnFailureIndex.class.getName() + "'");
+        indexName = createIndexAsync("CREATE CUSTOM INDEX ON %s (value) USING '" + WriteOnlyOnFailureIndex.class.getName() + "'");
         index = (WriteOnlyOnFailureIndex) getCurrentColumnFamilyStore().indexManager.getIndexByName(indexName);
-        assertTrue(waitForIndexBuilds(keyspace(), indexName));
+        waitForIndexBuilds(indexName);
         execute("INSERT INTO %s (pk, ck, value) VALUES (?, ?, ?)", 1, 1, 1);
         assertEquals(1, index.rowsInserted.size());
         assertInvalidThrow(IndexNotAvailableException.class, "SELECT value FROM %s WHERE value = 1");
@@ -1449,7 +1444,7 @@ public class SecondaryIndexTest extends CQLTester
     public void testIndexOnFrozenUDT() throws Throwable
     {
         String type = createType("CREATE TYPE %s (a int)");
-        String tableName = createTable("CREATE TABLE %s (k int PRIMARY KEY, v frozen<" + type + ">)");
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v frozen<" + type + ">)");
 
         Object udt1 = userType("a", 1);
         Object udt2 = userType("a", 2);
@@ -1459,7 +1454,6 @@ public class SecondaryIndexTest extends CQLTester
 
         execute("INSERT INTO %s (k, v) VALUES (?, ?)", 1, udt2);
         execute("INSERT INTO %s (k, v) VALUES (?, ?)", 1, udt1);
-        assertTrue(waitForIndex(keyspace(), tableName, indexName));
 
         assertRows(execute("SELECT * FROM %s WHERE v = ?", udt1), row(1, udt1), row(0, udt1));
         assertEmpty(execute("SELECT * FROM %s WHERE v = ?", udt2));
@@ -1478,7 +1472,7 @@ public class SecondaryIndexTest extends CQLTester
     public void testIndexOnFrozenCollectionOfUDT() throws Throwable
     {
         String type = createType("CREATE TYPE %s (a int)");
-        String tableName = createTable("CREATE TABLE %s (k int PRIMARY KEY, v frozen<set<frozen<" + type + ">>>)");
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v frozen<set<frozen<" + type + ">>>)");
 
         Object udt1 = userType("a", 1);
         Object udt2 = userType("a", 2);
@@ -1489,7 +1483,6 @@ public class SecondaryIndexTest extends CQLTester
         String indexName = createIndex("CREATE INDEX ON %s (full(v))");
 
         execute("INSERT INTO %s (k, v) VALUES (?, ?)", 2, set(udt2));
-        assertTrue(waitForIndex(keyspace(), tableName, indexName));
 
         assertInvalidMessage(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE,
                              "SELECT * FROM %s WHERE v CONTAINS ?", udt1);
@@ -1511,7 +1504,7 @@ public class SecondaryIndexTest extends CQLTester
     public void testIndexOnNonFrozenCollectionOfFrozenUDT() throws Throwable
     {
         String type = createType("CREATE TYPE %s (a int)");
-        String tableName = createTable("CREATE TABLE %s (k int PRIMARY KEY, v set<frozen<" + type + ">>)");
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v set<frozen<" + type + ">>)");
 
         Object udt1 = userType("a", 1);
         Object udt2 = userType("a", 2);
@@ -1525,7 +1518,6 @@ public class SecondaryIndexTest extends CQLTester
 
         execute("INSERT INTO %s (k, v) VALUES (?, ?)", 2, set(udt2));
         execute("UPDATE %s SET v = v + ? WHERE k = ?", set(udt2), 1);
-        assertTrue(waitForIndex(keyspace(), tableName, indexName));
 
         assertRows(execute("SELECT * FROM %s WHERE v CONTAINS ?", udt1), row(1, set(udt1, udt2)));
         assertRows(execute("SELECT * FROM %s WHERE v CONTAINS ?", udt2), row(1, set(udt1, udt2)), row(2, set(udt2)));
diff --git a/test/unit/org/apache/cassandra/cql3/validation/operations/InsertUpdateIfConditionTest.java b/test/unit/org/apache/cassandra/cql3/validation/operations/InsertUpdateIfConditionTest.java
index c617544684..531c1f6c02 100644
--- a/test/unit/org/apache/cassandra/cql3/validation/operations/InsertUpdateIfConditionTest.java
+++ b/test/unit/org/apache/cassandra/cql3/validation/operations/InsertUpdateIfConditionTest.java
@@ -466,8 +466,6 @@ public class InsertUpdateIfConditionTest extends CQLTester
         // create and confirm
         createIndex("CREATE INDEX IF NOT EXISTS myindex ON %s (value1)");
 
-        assertTrue(waitForIndex(KEYSPACE, tableName, "myindex"));
-
         // unsuccessful create since it's already there
         execute("CREATE INDEX IF NOT EXISTS myindex ON %s (value1)");
 
diff --git a/test/unit/org/apache/cassandra/db/guardrails/GuardrailTablesTest.java b/test/unit/org/apache/cassandra/db/guardrails/GuardrailTablesTest.java
index 7e69636600..d3d7ce023c 100644
--- a/test/unit/org/apache/cassandra/db/guardrails/GuardrailTablesTest.java
+++ b/test/unit/org/apache/cassandra/db/guardrails/GuardrailTablesTest.java
@@ -77,7 +77,7 @@ public class GuardrailTablesTest extends ThresholdTester
     @Override
     protected void dropTable(String tableName)
     {
-        dropFormattedTable(format("DROP TABLE %s.%s", keyspace(), tableName));
+        super.dropTable(format("DROP TABLE %s.%s", keyspace(), tableName));
     }
 
     private String assertCreateTableValid() throws Throwable
diff --git a/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java b/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java
index 0ab2f82440..d63d63840c 100644
--- a/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java
+++ b/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java
@@ -57,22 +57,20 @@ public class SecondaryIndexManagerTest extends CQLTester
     }
 
     @Test
-    public void creatingIndexMarksTheIndexAsBuilt() throws Throwable
+    public void creatingIndexMarksTheIndexAsBuilt()
     {
-        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
         String indexName = createIndex("CREATE INDEX ON %s(c)");
 
-        waitForIndex(KEYSPACE, tableName, indexName);
         assertMarkedAsBuilt(indexName);
     }
 
     @Test
     public void rebuilOrRecoveringIndexMarksTheIndexAsBuilt() throws Throwable
     {
-        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
         String indexName = createIndex("CREATE INDEX ON %s(c)");
 
-        waitForIndex(KEYSPACE, tableName, indexName);
         assertMarkedAsBuilt(indexName);
         
         assertTrue(tryRebuild(indexName, false));
@@ -82,10 +80,9 @@ public class SecondaryIndexManagerTest extends CQLTester
     @Test
     public void recreatingIndexMarksTheIndexAsBuilt() throws Throwable
     {
-        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
         String indexName = createIndex("CREATE INDEX ON %s(c)");
 
-        waitForIndex(KEYSPACE, tableName, indexName);
         assertMarkedAsBuilt(indexName);
 
         // drop the index and verify that it has been removed from the built indexes table
@@ -94,17 +91,15 @@ public class SecondaryIndexManagerTest extends CQLTester
 
         // create the index again and verify that it's added to the built indexes table
         createIndex(String.format("CREATE INDEX %s ON %%s(c)", indexName));
-        waitForIndex(KEYSPACE, tableName, indexName);
         assertMarkedAsBuilt(indexName);
     }
 
     @Test
-    public void addingSSTablesMarksTheIndexAsBuilt() throws Throwable
+    public void addingSSTablesMarksTheIndexAsBuilt()
     {
-        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
         String indexName = createIndex("CREATE INDEX ON %s(c)");
 
-        waitForIndex(KEYSPACE, tableName, indexName);
         assertMarkedAsBuilt(indexName);
 
         ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
@@ -123,10 +118,10 @@ public class SecondaryIndexManagerTest extends CQLTester
     {
         // create an index which blocks on creation
         TestingIndex.blockCreate();
-        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
-        String defaultIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
-        String readOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(b) USING '%s'", ReadOnlyOnFailureIndex.class.getName()));
-        String writeOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(b) USING '%s'", WriteOnlyOnFailureIndex.class.getName()));
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        String defaultIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
+        String readOnlyIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(b) USING '%s'", ReadOnlyOnFailureIndex.class.getName()));
+        String writeOnlyIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(b) USING '%s'", WriteOnlyOnFailureIndex.class.getName()));
 
         // try to rebuild/recover the index before the index creation task has finished
         assertFalse(tryRebuild(defaultIndexName, false));
@@ -138,9 +133,9 @@ public class SecondaryIndexManagerTest extends CQLTester
 
         // check that the index is marked as built when the creation finishes
         TestingIndex.unblockCreate();
-        waitForIndex(KEYSPACE, tableName, defaultIndexName);
-        waitForIndex(KEYSPACE, tableName, readOnlyIndexName);
-        waitForIndex(KEYSPACE, tableName, writeOnlyIndexName);
+        waitForIndexQueryable(defaultIndexName);
+        waitForIndexQueryable(readOnlyIndexName);
+        waitForIndexQueryable(writeOnlyIndexName);
         assertMarkedAsBuilt(defaultIndexName);
         assertMarkedAsBuilt(readOnlyIndexName);
         assertMarkedAsBuilt(writeOnlyIndexName);
@@ -157,16 +152,13 @@ public class SecondaryIndexManagerTest extends CQLTester
     @Test
     public void cannotRebuildOrRecoverWhileAnotherRebuildIsInProgress() throws Throwable
     {
-        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
         String defaultIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
         String readOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(b) USING '%s'", ReadOnlyOnFailureIndex.class.getName()));
         String writeOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(b) USING '%s'", WriteOnlyOnFailureIndex.class.getName()));
         final AtomicBoolean error = new AtomicBoolean();
 
-        // wait for index initialization and verify it's built:
-        waitForIndex(KEYSPACE, tableName, defaultIndexName);
-        waitForIndex(KEYSPACE, tableName, readOnlyIndexName);
-        waitForIndex(KEYSPACE, tableName, writeOnlyIndexName);
+        // verify it's built after initialization:
         assertMarkedAsBuilt(defaultIndexName);
         assertMarkedAsBuilt(readOnlyIndexName);
         assertMarkedAsBuilt(writeOnlyIndexName);
@@ -214,12 +206,11 @@ public class SecondaryIndexManagerTest extends CQLTester
     @Test
     public void cannotRebuildWhileAnSSTableBuildIsInProgress() throws Throwable
     {
-        final String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
         final String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
         final AtomicBoolean error = new AtomicBoolean();
 
-        // wait for index initialization and verify it's built:
-        waitForIndex(KEYSPACE, tableName, indexName);
+        // verify it's built after initialization:
         assertMarkedAsBuilt(indexName);
 
         // add sstables in another thread, but make it block:
@@ -260,12 +251,11 @@ public class SecondaryIndexManagerTest extends CQLTester
     @Test
     public void addingSSTableWhileRebuildIsInProgress() throws Throwable
     {
-        final String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
         final String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
         final AtomicBoolean error = new AtomicBoolean();
 
-        // wait for index initialization and verify it's built:
-        waitForIndex(KEYSPACE, tableName, indexName);
+        // verify it's built after initialization:
         assertMarkedAsBuilt(indexName);
 
         // rebuild the index in another thread, but make it block:
@@ -308,12 +298,11 @@ public class SecondaryIndexManagerTest extends CQLTester
     @Test
     public void addingSSTableWithBuildFailureWhileRebuildIsInProgress() throws Throwable
     {
-        final String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
         final String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
         final AtomicBoolean error = new AtomicBoolean();
 
-        // wait for index initialization and verify it's built:
-        waitForIndex(KEYSPACE, tableName, indexName);
+        // verify it's built after initialization:
         assertMarkedAsBuilt(indexName);
 
         // rebuild the index in another thread, but make it block:
@@ -362,11 +351,10 @@ public class SecondaryIndexManagerTest extends CQLTester
     }
 
     @Test
-    public void rebuildWithFailure() throws Throwable
+    public void rebuildWithFailure()
     {
-        final String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
         final String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
-        waitForIndex(KEYSPACE, tableName, indexName);
 
         // Rebuild the index with failure and verify it is not marked as built
         TestingIndex.shouldFailBuild = true;
@@ -384,13 +372,13 @@ public class SecondaryIndexManagerTest extends CQLTester
     }
 
     @Test
-    public void initializingIndexNotQueryableButMaybeWritable() throws Throwable
+    public void initializingIndexNotQueryableButMaybeWritable()
     {
         TestingIndex.blockCreate();
-        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
-        String defaultIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
-        String readOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", ReadOnlyOnFailureIndex.class.getName()));
-        String writeOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", WriteOnlyOnFailureIndex.class.getName()));
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        String defaultIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
+        String readOnlyIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", ReadOnlyOnFailureIndex.class.getName()));
+        String writeOnlyIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", WriteOnlyOnFailureIndex.class.getName()));
 
         // the index shouldn't be queryable while the initialization hasn't finished
         assertFalse(isQueryable(defaultIndexName));
@@ -402,9 +390,9 @@ public class SecondaryIndexManagerTest extends CQLTester
 
         // the index should be queryable once the initialization has finished
         TestingIndex.unblockCreate();
-        waitForIndex(KEYSPACE, tableName, defaultIndexName);
-        waitForIndex(KEYSPACE, tableName, readOnlyIndexName);
-        waitForIndex(KEYSPACE, tableName, writeOnlyIndexName);
+        waitForIndexQueryable(defaultIndexName);
+        waitForIndexQueryable(readOnlyIndexName);
+        waitForIndexQueryable(writeOnlyIndexName);
         assertTrue(isQueryable(defaultIndexName));
         assertTrue(isQueryable(readOnlyIndexName));
         assertTrue(isQueryable(writeOnlyIndexName));
@@ -414,13 +402,13 @@ public class SecondaryIndexManagerTest extends CQLTester
     }
 
     @Test
-    public void initializingIndexNotQueryableButMaybeNotWritableAfterPartialRebuild() throws Throwable
+    public void initializingIndexNotQueryableButMaybeNotWritableAfterPartialRebuild()
     {
         TestingIndex.blockCreate();
-        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
-        String defaultIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
-        String readOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", ReadOnlyOnFailureIndex.class.getName()));
-        String writeOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", WriteOnlyOnFailureIndex.class.getName()));
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        String defaultIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
+        String readOnlyIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", ReadOnlyOnFailureIndex.class.getName()));
+        String writeOnlyIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", WriteOnlyOnFailureIndex.class.getName()));
 
         // the index should never be queryable while the initialization hasn't finished
         assertFalse(isQueryable(defaultIndexName));
@@ -463,7 +451,7 @@ public class SecondaryIndexManagerTest extends CQLTester
 
         // the index should be queryable once the initialization has finished
         TestingIndex.unblockCreate();
-        waitForIndex(KEYSPACE, tableName, defaultIndexName);
+        waitForIndexQueryable(defaultIndexName);
         assertTrue(isQueryable(defaultIndexName));
         assertTrue(isQueryable(readOnlyIndexName));
         assertTrue(isQueryable(writeOnlyIndexName));
@@ -477,12 +465,12 @@ public class SecondaryIndexManagerTest extends CQLTester
     {
         TestingIndex.shouldFailCreate = true;
         createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
-        String defaultIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
-        String readOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", ReadOnlyOnFailureIndex.class.getName()));
-        String writeOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", WriteOnlyOnFailureIndex.class.getName()));
-        assertTrue(waitForIndexBuilds(KEYSPACE, defaultIndexName));
-        assertTrue(waitForIndexBuilds(KEYSPACE, readOnlyIndexName));
-        assertTrue(waitForIndexBuilds(KEYSPACE, writeOnlyIndexName));
+        String defaultIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
+        String readOnlyIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", ReadOnlyOnFailureIndex.class.getName()));
+        String writeOnlyIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", WriteOnlyOnFailureIndex.class.getName()));
+        waitForIndexBuilds(defaultIndexName);
+        waitForIndexBuilds(readOnlyIndexName);
+        waitForIndexBuilds(writeOnlyIndexName);
 
         tryRebuild(defaultIndexName, true);
         tryRebuild(readOnlyIndexName, true);
@@ -501,16 +489,16 @@ public class SecondaryIndexManagerTest extends CQLTester
     }
 
     @Test
-    public void indexWithFailedInitializationDoesNotChangeQueryabilityNorWritabilityAfterPartialRebuild() throws Throwable
+    public void indexWithFailedInitializationDoesNotChangeQueryabilityNorWritabilityAfterPartialRebuild()
     {
         TestingIndex.shouldFailCreate = true;
         createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
-        String defaultIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
-        String readOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", ReadOnlyOnFailureIndex.class.getName()));
-        String writeOnlyIndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", WriteOnlyOnFailureIndex.class.getName()));
-        assertTrue(waitForIndexBuilds(KEYSPACE, defaultIndexName));
-        assertTrue(waitForIndexBuilds(KEYSPACE, readOnlyIndexName));
-        assertTrue(waitForIndexBuilds(KEYSPACE, writeOnlyIndexName));
+        String defaultIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
+        String readOnlyIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", ReadOnlyOnFailureIndex.class.getName()));
+        String writeOnlyIndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", WriteOnlyOnFailureIndex.class.getName()));
+        waitForIndexBuilds(defaultIndexName);
+        waitForIndexBuilds(readOnlyIndexName);
+        waitForIndexBuilds(writeOnlyIndexName);
         TestingIndex.shouldFailCreate = false;
 
         // the index should never be queryable, but it could be writable after the failed initialization
@@ -524,9 +512,9 @@ public class SecondaryIndexManagerTest extends CQLTester
         // a successful partial build doesn't set the index as queryable nor writable
         ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
         cfs.indexManager.handleNotification(new SSTableAddedNotification(cfs.getLiveSSTables(), null), this);
-        assertTrue(waitForIndexBuilds(KEYSPACE, defaultIndexName));
-        assertTrue(waitForIndexBuilds(KEYSPACE, readOnlyIndexName));
-        assertTrue(waitForIndexBuilds(KEYSPACE, writeOnlyIndexName));
+        waitForIndexBuilds(defaultIndexName);
+        waitForIndexBuilds(readOnlyIndexName);
+        waitForIndexBuilds(writeOnlyIndexName);
         assertFalse(isQueryable(defaultIndexName));
         assertFalse(isQueryable(readOnlyIndexName));
         assertFalse(isQueryable(writeOnlyIndexName));
@@ -556,7 +544,7 @@ public class SecondaryIndexManagerTest extends CQLTester
             TestingIndex.failedCreateThrowable = throwable;
 
             createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
-            String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
+            String indexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
             tryRebuild(indexName, true);
             fail("Should have failed!");
         }
@@ -573,7 +561,7 @@ public class SecondaryIndexManagerTest extends CQLTester
     }
 
     @Test
-    public void handleJVMStablityOnFailedRebuild() throws Throwable
+    public void handleJVMStablityOnFailedRebuild()
     {
         handleJVMStablityOnFailedRebuild(new SocketException("Should not fail"), false);
         handleJVMStablityOnFailedRebuild(new FileNotFoundException("Should not fail"), false);
@@ -582,11 +570,10 @@ public class SecondaryIndexManagerTest extends CQLTester
         handleJVMStablityOnFailedRebuild(new RuntimeException("Should not fail"), false);
     }
 
-    private void handleJVMStablityOnFailedRebuild(Throwable throwable, boolean shouldKillJVM) throws Throwable
+    private void handleJVMStablityOnFailedRebuild(Throwable throwable, boolean shouldKillJVM)
     {
-        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
         String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
-        waitForIndex(KEYSPACE, tableName, indexName);
 
         KillerForTests killerForTests = new KillerForTests();
         JVMStabilityInspector.Killer originalKiller = JVMStabilityInspector.replaceKiller(killerForTests);
diff --git a/test/unit/org/apache/cassandra/index/internal/CassandraIndexTest.java b/test/unit/org/apache/cassandra/index/internal/CassandraIndexTest.java
index 84eb8d0746..292b25a7d0 100644
--- a/test/unit/org/apache/cassandra/index/internal/CassandraIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/internal/CassandraIndexTest.java
@@ -50,7 +50,6 @@ import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 /**
@@ -426,7 +425,7 @@ public class CassandraIndexTest extends CQLTester
         Object[] row2 = row("k0", "c1");
         Object[] row3 = row("k1", "c0");
         Object[] row4 = row("k1", "c1");
-        String tableName = createTable("CREATE TABLE %s (k text, c text, PRIMARY KEY(k, c))");
+        createTable("CREATE TABLE %s (k text, c text, PRIMARY KEY(k, c))");
         createIndex("CREATE INDEX no_regulars_idx ON %s(c)");
 
         execute("INSERT INTO %s (k, c) VALUES (?, ?)", row1);
@@ -440,7 +439,6 @@ public class CassandraIndexTest extends CQLTester
 
         dropIndex("DROP INDEX %s.no_regulars_idx");
         createIndex("CREATE INDEX no_regulars_idx ON %s(c)");
-        assertTrue(waitForIndex(keyspace(), tableName, "no_regulars_idx"));
 
         assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE c = ?", "c0"), row1, row3);
         assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE c = ?", "c1"), row2, row4);
@@ -572,16 +570,15 @@ public class CassandraIndexTest extends CQLTester
                   .untilAsserted(() -> assertRows(execute(selectBuiltIndexesQuery), row("system", "PaxosUncommittedIndex", null)));
 
         String indexName = "build_remove_test_idx";
-        String tableName = createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
+        createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
         createIndex(String.format("CREATE INDEX %s ON %%s(c)", indexName));
-        waitForIndex(KEYSPACE, tableName, indexName);
 
         // check that there are no other rows in the built indexes table
         assertRows(execute(selectBuiltIndexesQuery), row(KEYSPACE, indexName, null), row("system", "PaxosUncommittedIndex", null));
 
         // rebuild the index and verify the built status table
         getCurrentColumnFamilyStore().rebuildSecondaryIndex(indexName);
-        waitForIndex(KEYSPACE, tableName, indexName);
+        waitForIndexQueryable(indexName);
 
         // check that there are no other rows in the built indexes table
         assertRows(execute(selectBuiltIndexesQuery), row(KEYSPACE, indexName, null), row("system", "PaxosUncommittedIndex", null));
diff --git a/test/unit/org/apache/cassandra/index/sai/SAITester.java b/test/unit/org/apache/cassandra/index/sai/SAITester.java
index fe9360e7ee..dc963153ae 100644
--- a/test/unit/org/apache/cassandra/index/sai/SAITester.java
+++ b/test/unit/org/apache/cassandra/index/sai/SAITester.java
@@ -97,7 +97,6 @@ import org.apache.cassandra.service.snapshot.TableSnapshot;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.Throwables;
 import org.apache.lucene.codecs.CodecUtil;
-import org.awaitility.Awaitility;
 
 import static org.apache.cassandra.inject.ActionBuilder.newActionBuilder;
 import static org.apache.cassandra.inject.Expression.quote;
@@ -119,8 +118,6 @@ public abstract class SAITester extends CQLTester
 
     protected static final ColumnIdentifier V1_COLUMN_IDENTIFIER = ColumnIdentifier.getInterned("v1", true);
 
-    protected static final int ASSERTION_TIMEOUT_SECONDS = 15;
-
     protected static final Injections.Counter indexBuildCounter = Injections.newCounter("IndexBuildCounter")
                                                                             .add(newInvokePoint().onClass(CompactionManager.class)
                                                                                                  .onMethod("submitIndexBuild",
@@ -273,7 +270,7 @@ public abstract class SAITester extends CQLTester
         cfs.indexManager.executePreJoinTasksBlocking(true);
         if (wait)
         {
-            waitForIndexQueryable();
+            waitForTableIndexesQueryable();
         }
     }
 
@@ -297,38 +294,12 @@ public abstract class SAITester extends CQLTester
         }
     }
 
-    protected void waitForAssert(Runnable runnableAssert, long timeout, TimeUnit unit)
-    {
-        Awaitility.await().dontCatchUncaughtExceptions().atMost(timeout, unit).untilAsserted(runnableAssert::run);
-    }
-
-    protected void waitForAssert(Runnable assertion)
-    {
-        waitForAssert(assertion, ASSERTION_TIMEOUT_SECONDS, TimeUnit.SECONDS);
-    }
-
     protected boolean indexNeedsFullRebuild(String index)
     {
         ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
         return cfs.indexManager.needsFullRebuild(index);
     }
 
-    protected boolean isIndexQueryable()
-    {
-        return isIndexQueryable(KEYSPACE, currentTable());
-    }
-
-    protected boolean isIndexQueryable(String keyspace, String table)
-    {
-        ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
-        for (Index index : cfs.indexManager.listIndexes())
-        {
-            if (!cfs.indexManager.isIndexQueryable(index))
-                return false;
-        }
-        return true;
-    }
-
     protected void verifyInitialIndexFailed(String indexName)
     {
         // Verify that the initial index build fails...
@@ -388,16 +359,6 @@ public abstract class SAITester extends CQLTester
         return metricValue;
     }
 
-    public void waitForIndexQueryable()
-    {
-        waitForIndexQueryable(KEYSPACE, currentTable());
-    }
-
-    public void waitForIndexQueryable(String keyspace, String table)
-    {
-        waitForAssert(() -> assertTrue(isIndexQueryable(keyspace, table)), 60, TimeUnit.SECONDS);
-    }
-
     public void waitForCompactions()
     {
         waitForAssert(() -> assertFalse(CompactionManager.instance.isCompacting(ColumnFamilyStore.all(), ssTableReader -> true)), 10, TimeUnit.SECONDS);
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/AllowFilteringTest.java b/test/unit/org/apache/cassandra/index/sai/cql/AllowFilteringTest.java
index 13b0f6148b..f80d07eeb7 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/AllowFilteringTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/AllowFilteringTest.java
@@ -38,7 +38,6 @@ public class AllowFilteringTest extends SAITester
         createTable("CREATE TABLE %s (k1 int, k2 int, c1 int, c2 int, c3 int, v1 int, " +
                     "PRIMARY KEY ((k1, k2), c1, c2, c3))");
         createIndex(format("CREATE CUSTOM INDEX ON %%s(c1) USING '%s'", StorageAttachedIndex.class.getName()));
-        waitForIndexQueryable();
 
         // with only index restrictions
         test("SELECT * FROM %s WHERE c1=0", false);
@@ -83,7 +82,6 @@ public class AllowFilteringTest extends SAITester
         createTable("CREATE TABLE %s (k1 int, k2 int, c1 int, c2 int, c3 int, c4 int, v1 int, " +
                     "PRIMARY KEY ((k1, k2), c1, c2, c3, c4))");
         createIndex(format("CREATE CUSTOM INDEX ON %%s(c3) USING '%s'", StorageAttachedIndex.class.getName()));
-        waitForIndexQueryable();
 
         // with only index restrictions
         test("SELECT * FROM %s WHERE c3=0", false);
@@ -129,7 +127,6 @@ public class AllowFilteringTest extends SAITester
                     "PRIMARY KEY ((k1, k2), c1, c2, c3, c4))");
         createIndex(format("CREATE CUSTOM INDEX ON %%s(c2) USING '%s'", StorageAttachedIndex.class.getName()));
         createIndex(format("CREATE CUSTOM INDEX ON %%s(c4) USING '%s'", StorageAttachedIndex.class.getName()));
-        waitForIndexQueryable();
 
         // with only index restrictions
         test("SELECT * FROM %s WHERE c2=0 AND c4=0", false);
@@ -175,7 +172,6 @@ public class AllowFilteringTest extends SAITester
     {
         createTable("CREATE TABLE %s (k1 int, k2 int, c1 int, c2 int, v1 int, v2 int, PRIMARY KEY ((k1, k2), c1, c2))");
         createIndex(format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName()));
-        waitForIndexQueryable();
 
         // with only index restrictions
         test("SELECT * FROM %s WHERE v1=0", false);
@@ -219,7 +215,6 @@ public class AllowFilteringTest extends SAITester
                     "PRIMARY KEY ((k1, k2), c1, c2))");
         createIndex(format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName()));
         createIndex(format("CREATE CUSTOM INDEX ON %%s(v2) USING '%s'", StorageAttachedIndex.class.getName()));
-        waitForIndexQueryable();
 
         // with only index restrictions
         test("SELECT * FROM %s WHERE v1=0 AND v2=0", false);
@@ -268,7 +263,6 @@ public class AllowFilteringTest extends SAITester
         createIndex(format("CREATE CUSTOM INDEX ON %%s(c4) USING '%s'", StorageAttachedIndex.class.getName()));
         createIndex(format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName()));
         createIndex(format("CREATE CUSTOM INDEX ON %%s(v2) USING '%s'", StorageAttachedIndex.class.getName()));
-        waitForIndexQueryable();
 
         // with only index restrictions
         test("SELECT * FROM %s WHERE c2=0 AND c4=0 AND v1=0 AND v2=0", false);
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/BaseDataModel.java b/test/unit/org/apache/cassandra/index/sai/cql/BaseDataModel.java
index 67621c3b9a..e9287e178f 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/BaseDataModel.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/BaseDataModel.java
@@ -169,14 +169,22 @@ class BaseDataModel
 
     public void createIndexes(Executor tester) throws Throwable
     {
-        String template = "CREATE CUSTOM INDEX ndi_%s_index_%s ON %%s (%s) USING 'StorageAttachedIndex'";
+        createIndexes(tester, columns);
+    }
+
+    protected void createIndexes(Executor tester, List<Pair<String, String>> columns) throws Throwable
+    {
+        String indexNameTemplate = "sai_%s_index_%s";
+        String createIndexTemplate = "CREATE CUSTOM INDEX %s ON %%s (%s) USING 'StorageAttachedIndex'";
 
         for (Pair<String, String> column : columns)
         {
-            if (!skipColumns.contains(column.left))
+            String columnName = column.left;
+            if (!skipColumns.contains(columnName))
             {
-                executeLocalIndexed(tester, String.format(template, column.left, indexedTable, column.left));
-                tester.waitForIndexQueryable(KEYSPACE, indexedTable);
+                String indexName = String.format(indexNameTemplate, columnName, indexedTable);
+                executeLocalIndexed(tester, String.format(createIndexTemplate, indexName, columnName));
+                tester.waitForIndexQueryable(KEYSPACE, indexName);
             }
         }
     }
@@ -294,16 +302,7 @@ class BaseDataModel
         public void createIndexes(Executor tester) throws Throwable
         {
             super.createIndexes(tester);
-            String template = "CREATE CUSTOM INDEX ndi_%s_index_%s ON %%s (%s) USING 'StorageAttachedIndex'";
-
-            for (Pair<String, String> column : keyColumns)
-            {
-                if (!skipColumns.contains(column.left))
-                {
-                    executeLocalIndexed(tester, String.format(template, column.left, indexedTable, column.left));
-                    tester.waitForIndexQueryable(KEYSPACE, indexedTable);
-                }
-            }
+            createIndexes(tester, keyColumns);
         }
 
         @Override
@@ -383,7 +382,7 @@ class BaseDataModel
     {
         void createTable(String statement);
 
-        void waitForIndexQueryable(String keyspace, String table);
+        void waitForIndexQueryable(String keyspace, String index);
 
         void executeLocal(String query, Object...values) throws Throwable;
 
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/BooleanTypeTest.java b/test/unit/org/apache/cassandra/index/sai/cql/BooleanTypeTest.java
index 3131414418..75e72b1686 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/BooleanTypeTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/BooleanTypeTest.java
@@ -32,7 +32,6 @@ public class BooleanTypeTest extends SAITester
         createTable("CREATE TABLE %s (id text PRIMARY KEY, val boolean)");
 
         createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
-        waitForIndexQueryable();
 
         execute("INSERT INTO %s (id, val) VALUES ('0', false)");
         execute("INSERT INTO %s (id, val) VALUES ('1', true)");
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/CollectionIndexingTest.java b/test/unit/org/apache/cassandra/index/sai/cql/CollectionIndexingTest.java
index a3906ab6dc..8bc9efb1ca 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/CollectionIndexingTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/CollectionIndexingTest.java
@@ -178,7 +178,6 @@ public class CollectionIndexingTest extends SAITester
     {
         createTable("CREATE TABLE %s (pk int primary key, value map<int, text>)");
         createIndex(createIndex);
-        waitForIndexQueryable();
         execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 1, new HashMap<Integer, String>() {{
             put(1, "v1");
             put(2, "v2");
@@ -194,7 +193,6 @@ public class CollectionIndexingTest extends SAITester
     {
         createTable("CREATE TABLE %s (pk int primary key, value frozen<map<int, text>>)");
         createIndex(createIndex);
-        waitForIndexQueryable();
         execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 1, new HashMap<Integer, String>() {{
             put(1, "v1");
             put(2, "v2");
@@ -210,7 +208,6 @@ public class CollectionIndexingTest extends SAITester
     {
         createTable("CREATE TABLE %s (pk int primary key, value frozen<list<int>>)");
         createIndex(createIndex);
-        waitForIndexQueryable();
         execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 1, Arrays.asList(1, 2, 3));
         execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 2, Arrays.asList(1, 2, 3));
         execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 3, Arrays.asList(4, 5, 6));
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/DecimalLargeValueTest.java b/test/unit/org/apache/cassandra/index/sai/cql/DecimalLargeValueTest.java
index 97a51827fd..107dae1c0a 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/DecimalLargeValueTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/DecimalLargeValueTest.java
@@ -37,8 +37,6 @@ public class DecimalLargeValueTest extends SAITester
 
         createIndex("CREATE CUSTOM INDEX ON %s(dec) USING 'StorageAttachedIndex'");
 
-        waitForIndexQueryable();
-
         disableCompaction();
     }
 
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/DuplicateRowIDTest.java b/test/unit/org/apache/cassandra/index/sai/cql/DuplicateRowIDTest.java
index 36da00b586..fd13e33473 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/DuplicateRowIDTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/DuplicateRowIDTest.java
@@ -40,7 +40,6 @@ public class DuplicateRowIDTest extends SAITester
     {
         createTable("CREATE TABLE %s (id1 TEXT PRIMARY KEY, v1 INT)");
         createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
-        waitForIndexQueryable();
 
         for (int i = 0; i < 2048; ++i)
         {
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/MixedIndexImplementationsTest.java b/test/unit/org/apache/cassandra/index/sai/cql/MixedIndexImplementationsTest.java
index f705240d68..cb3ef36e99 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/MixedIndexImplementationsTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/MixedIndexImplementationsTest.java
@@ -41,7 +41,6 @@ public class MixedIndexImplementationsTest extends SAITester
         String ossIndex = createIndex("CREATE INDEX ON %s(v1)");
         String ndiIndex = createIndex(
                 String.format("CREATE CUSTOM INDEX ON %%s(v2) USING '%s'", StorageAttachedIndex.class.getName()));
-        waitForIndexQueryable();
 
         // drop non-storage-attached index when a SAI index exists
         dropIndex("DROP INDEX %s." + ossIndex);
@@ -68,8 +67,6 @@ public class MixedIndexImplementationsTest extends SAITester
         execute(insert, 2, 1, 0);
         execute(insert, 3, 1, 1);
 
-        waitForIndexQueryable();
-
         String ossSelect = "SELECT * FROM %s WHERE v1 = ?";
         assertRowsIgnoringOrder(execute(ossSelect, 0), new Object[][]{{0, 0, 0}, {1, 0, 1}});
         assertRowsIgnoringOrder(execute(ossSelect, 1), new Object[][]{{2, 1, 0}, {3, 1, 1}});
@@ -99,7 +96,6 @@ public class MixedIndexImplementationsTest extends SAITester
         createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c3) USING '%s'", StorageAttachedIndex.class.getName()));
         createIndex(String.format("CREATE CUSTOM INDEX ON %%s(r1) USING '%s'", StorageAttachedIndex.class.getName()));
         createIndex(String.format("CREATE CUSTOM INDEX ON %%s(r2) USING '%s'", StorageAttachedIndex.class.getName()));
-        waitForIndexQueryable();
 
         // without using the not-SAI index
         testAllowFiltering("SELECT * FROM %s", false);
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/MultipleColumnIndexTest.java b/test/unit/org/apache/cassandra/index/sai/cql/MultipleColumnIndexTest.java
index c29b909c8c..97f93e29aa 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/MultipleColumnIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/MultipleColumnIndexTest.java
@@ -46,8 +46,6 @@ public class MultipleColumnIndexTest extends SAITester
         createIndex("CREATE CUSTOM INDEX ON %s(values(text_map)) USING 'StorageAttachedIndex'");
         createIndex("CREATE CUSTOM INDEX ON %s(entries(text_map)) USING 'StorageAttachedIndex'");
 
-        waitForIndexQueryable();
-
         execute("INSERT INTO %s(id, text_map) values (1, {'k1':'v1', 'k2':'v2'})");
         execute("INSERT INTO %s(id, text_map) values (2, {'k1':'v1', 'k3':'v3'})");
         execute("INSERT INTO %s(id, text_map) values (3, {'k4':'v4', 'k5':'v5'})");
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/SingleNodeExecutor.java b/test/unit/org/apache/cassandra/index/sai/cql/SingleNodeExecutor.java
index 54f96bc6d8..1e86eb4c15 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/SingleNodeExecutor.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/SingleNodeExecutor.java
@@ -43,9 +43,9 @@ public class SingleNodeExecutor implements BaseDataModel.Executor
     }
 
     @Override
-    public void waitForIndexQueryable(String keyspace, String table)
+    public void waitForIndexQueryable(String keyspace, String index)
     {
-        tester.waitForIndexQueryable(keyspace, table);
+        tester.waitForIndexQueryable(keyspace, index);
     }
 
     @Override
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/StorageAttachedIndexDDLTest.java b/test/unit/org/apache/cassandra/index/sai/cql/StorageAttachedIndexDDLTest.java
index a828592374..52d364c210 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/StorageAttachedIndexDDLTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/StorageAttachedIndexDDLTest.java
@@ -70,6 +70,7 @@ import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.Throwables;
+import org.assertj.core.api.Assertions;
 import org.mockito.Mockito;
 
 import static java.util.Collections.singletonList;
@@ -257,7 +258,7 @@ public class StorageAttachedIndexDDLTest extends SAITester
 
         createIndex("CREATE CUSTOM INDEX IF NOT EXISTS ON %s(val) USING 'StorageAttachedIndex' ");
 
-        createIndex("CREATE CUSTOM INDEX IF NOT EXISTS ON %s(val) USING 'StorageAttachedIndex' ");
+        createIndexAsync("CREATE CUSTOM INDEX IF NOT EXISTS ON %s(val) USING 'StorageAttachedIndex' ");
 
         assertEquals(1, saiCreationCounter.get());
     }
@@ -334,7 +335,7 @@ public class StorageAttachedIndexDDLTest extends SAITester
         flush();
 
         executeNet("CREATE CUSTOM INDEX index_1 ON %s(v1) USING 'StorageAttachedIndex'");
-        waitForIndexQueryable();
+        waitForTableIndexesQueryable();
 
         // same name
         assertThatThrownBy(() -> executeNet("CREATE CUSTOM INDEX index_1 ON %s(v1) USING 'StorageAttachedIndex'"))
@@ -365,7 +366,6 @@ public class StorageAttachedIndexDDLTest extends SAITester
             execute("INSERT INTO %s (id1, v1) VALUES ('" + i + "', '0')");
 
         createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
-        waitForIndexQueryable();
 
         ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1='0'");
         assertEquals(rowCount, rows.all().size());
@@ -384,7 +384,7 @@ public class StorageAttachedIndexDDLTest extends SAITester
 
         // Create the index, but do not allow the initial index build to begin:
         Injections.inject(delayInitializationTask);
-        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        createIndexAsync("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
 
         // Flush the Memtable's contents, which will feed data to the index as the SSTable is written:
         flush();
@@ -392,7 +392,7 @@ public class StorageAttachedIndexDDLTest extends SAITester
         // Allow the initialization task, which builds the index, to continue:
         delayInitializationTask.countDown();
 
-        waitForIndexQueryable();
+        waitForTableIndexesQueryable();
 
         ResultSet rows = executeNet("SELECT id FROM %s WHERE val = 'Camel'");
         assertEquals(1, rows.all().size());
@@ -410,7 +410,7 @@ public class StorageAttachedIndexDDLTest extends SAITester
             execute("INSERT INTO %s (id1, v1) VALUES ('" + i + "', '0')");
 
         Injections.inject(forceFlushPause);
-        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+        createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1"));
 
         assertThatThrownBy(() -> executeNet("SELECT id1 FROM %s WHERE v1='0'")).isInstanceOf(ReadFailureException.class);
     }
@@ -427,7 +427,7 @@ public class StorageAttachedIndexDDLTest extends SAITester
         flush();
 
         Injections.inject(failSAIInitialializaion);
-        createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+        createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1"));
         waitForAssert(() -> assertEquals(1, indexBuildCounter.get()));
         waitForCompactions();
 
@@ -478,7 +478,6 @@ public class StorageAttachedIndexDDLTest extends SAITester
         verifyNoIndexFiles();
 
         IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), UTF8Type.instance);
-        waitForIndexQueryable();
         verifyIndexFiles(literalIndexContext, 2);
         ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1='0'");
         assertEquals(2, rows.all().size());
@@ -501,7 +500,6 @@ public class StorageAttachedIndexDDLTest extends SAITester
         verifyNoIndexFiles();
 
         IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), UTF8Type.instance);
-        waitForIndexQueryable();
         verifyIndexFiles(literalIndexContext, 2);
         ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1='0'");
         assertEquals(2, rows.all().size());
@@ -571,9 +569,9 @@ public class StorageAttachedIndexDDLTest extends SAITester
 
         if (concurrentTruncate)
         {
-            createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+            createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1"));
             truncate(true);
-            waitForIndexQueryable();
+            waitForTableIndexesQueryable();
         }
         else
         {
@@ -721,7 +719,7 @@ public class StorageAttachedIndexDDLTest extends SAITester
         try
         {
             // Create a new index, which will actuate a build compaction and fail, but leave the node running...
-            IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), UTF8Type.instance);
+            IndexContext literalIndexContext = createIndexContext(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1")), UTF8Type.instance);
             // two index builders running in different compaction threads because of parallelised index initial build
             waitForAssert(() -> assertEquals(2, indexBuildCounter.get()));
             waitForCompactionsFinished();
@@ -729,7 +727,7 @@ public class StorageAttachedIndexDDLTest extends SAITester
             // Only token/primary key files for the first SSTable in the compaction task should exist, while column-specific files are blown away:
             verifyIndexFiles(literalIndexContext, 2, 0, 0);
 
-            assertFalse(isIndexQueryable());
+            Assertions.assertThat(getNotQueryableIndexes()).isNotEmpty();
 
             assertZeroSegmentBuilderUsage();
         }
@@ -757,7 +755,7 @@ public class StorageAttachedIndexDDLTest extends SAITester
         try
         {
             // Create a new index, which will actuate a build compaction and fail, but leave the node running...
-            createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+            createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1"));
             // two index builders running in different compaction threads because of parallelised index initial build
             waitForAssert(() -> assertEquals(2, indexBuildCounter.get()));
             waitForAssert(() -> assertEquals(0, getCompactionTasks()));
@@ -765,7 +763,7 @@ public class StorageAttachedIndexDDLTest extends SAITester
             // SSTable-level token/offset file(s) should be removed, while column-specific files never existed:
             verifyNoIndexFiles();
 
-            assertFalse(isIndexQueryable());
+            Assertions.assertThat(getNotQueryableIndexes()).isNotEmpty();
 
             assertZeroSegmentBuilderUsage();
         }
@@ -857,7 +855,6 @@ public class StorageAttachedIndexDDLTest extends SAITester
 
 
         IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), UTF8Type.instance);
-        waitForIndexQueryable();
 
         populateData.run();
         verifySSTableIndexes(IndexMetadata.generateDefaultIndexName(currentTable(), V1_COLUMN_IDENTIFIER), 2, 0);
@@ -895,7 +892,7 @@ public class StorageAttachedIndexDDLTest extends SAITester
                                                                    .build();
 
         Injections.inject(delayIndexBuilderCompletion);
-        String indexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+        String indexName = createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1"));
         waitForAssert(() -> assertEquals(1, delayIndexBuilderCompletion.getCount()));
 
         dropIndex("DROP INDEX %s." + indexName);
@@ -911,7 +908,6 @@ public class StorageAttachedIndexDDLTest extends SAITester
 
         // create index again, it should succeed
         indexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
-        waitForIndexQueryable();
         verifySSTableIndexes(indexName, 1);
 
         ResultSet rows = executeNet("SELECT id1 FROM %s WHERE v1='0'");
@@ -940,7 +936,7 @@ public class StorageAttachedIndexDDLTest extends SAITester
 
         Injections.inject(delayIndexBuilderCompletion);
 
-        IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), UTF8Type.instance);
+        IndexContext literalIndexContext = createIndexContext(createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1")), UTF8Type.instance);
 
         waitForAssert(() -> assertTrue(getCompactionTasks() > 0), 1000, TimeUnit.MILLISECONDS);
 
@@ -963,7 +959,7 @@ public class StorageAttachedIndexDDLTest extends SAITester
 
         // initial index builder should have stopped abruptly resulting in the index not being queryable
         verifyInitialIndexFailed(literalIndexContext.getIndexName());
-        assertFalse(isIndexQueryable());
+        Assertions.assertThat(getNotQueryableIndexes()).isNotEmpty();
 
         ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
         for (Index i : cfs.indexManager.listIndexes())
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/types/IndexingTypeSupport.java b/test/unit/org/apache/cassandra/index/sai/cql/types/IndexingTypeSupport.java
index 267d37c1c5..15acbdaa8d 100644
--- a/test/unit/org/apache/cassandra/index/sai/cql/types/IndexingTypeSupport.java
+++ b/test/unit/org/apache/cassandra/index/sai/cql/types/IndexingTypeSupport.java
@@ -97,7 +97,6 @@ public abstract class IndexingTypeSupport extends SAITester
         {
             for (String index : dataset.decorateIndexColumn("value"))
                 createIndex(String.format("CREATE CUSTOM INDEX ON %%s(%s) USING 'StorageAttachedIndex'", index));
-            waitForIndexQueryable();
         }
 
         insertData(this, allRows, scenario);
@@ -115,7 +114,6 @@ public abstract class IndexingTypeSupport extends SAITester
                 flush();
                 for (String index : dataset.decorateIndexColumn("value"))
                     createIndex(String.format("CREATE CUSTOM INDEX ON %%s(%s) USING 'StorageAttachedIndex'", index));
-                waitForIndexQueryable();
                 break;
         }
 
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/NodeStartupTest.java b/test/unit/org/apache/cassandra/index/sai/disk/NodeStartupTest.java
index 8b3b768e5e..6855b0eb92 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/NodeStartupTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/NodeStartupTest.java
@@ -47,6 +47,7 @@ import org.apache.cassandra.inject.Injection;
 import org.apache.cassandra.inject.Injections;
 import org.apache.cassandra.inject.InvokePointBuilder;
 import org.apache.cassandra.schema.Schema;
+import org.assertj.core.api.Assertions;
 
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -241,7 +242,7 @@ public class NodeStartupTest extends SAITester
     {
         populator.populate(this);
 
-        assertTrue(isIndexQueryable());
+        Assertions.assertThat(getNotQueryableIndexes()).isEmpty();
         assertTrue(isGroupIndexComplete());
         assertTrue(isColumnIndexComplete());
         Assert.assertEquals(expectedDocuments, execute("SELECT * FROM %s WHERE v1 = '0'").size());
@@ -252,7 +253,7 @@ public class NodeStartupTest extends SAITester
 
         simulateNodeRestart();
 
-        assertTrue(isIndexQueryable());
+        Assertions.assertThat(getNotQueryableIndexes()).isEmpty();
         assertTrue(isGroupIndexComplete());
         assertTrue(isColumnIndexComplete());
         Assert.assertEquals(expectedDocuments, execute("SELECT * FROM %s WHERE v1 = '0'").size());
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/SelectiveIntersectionTest.java b/test/unit/org/apache/cassandra/index/sai/disk/SelectiveIntersectionTest.java
index 349a24b98c..51c7637754 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/SelectiveIntersectionTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/SelectiveIntersectionTest.java
@@ -63,8 +63,6 @@ public class SelectiveIntersectionTest extends SAITester
         createIndex(String.format(CREATE_INDEX_TEMPLATE, "v2"));
         createIndex(String.format(CREATE_INDEX_TEMPLATE, "v3"));
 
-        waitForIndexQueryable();
-
         for (int i = 0; i < 100; ++i)
         {
             execute("INSERT INTO %s(pk,v1,v2,v3) VALUES (?, ?, ?, ?)", i, Integer.toString(i), Integer.toString(i / 20), Integer.toString(i % 10));
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/SingleNodeQueryFailureTest.java b/test/unit/org/apache/cassandra/index/sai/disk/SingleNodeQueryFailureTest.java
index a208aa9770..e49a9fe249 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/SingleNodeQueryFailureTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/SingleNodeQueryFailureTest.java
@@ -73,7 +73,6 @@ public class SingleNodeQueryFailureTest extends SAITester
 
         createTable(CREATE_TABLE_TEMPLATE);
         createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
-        waitForIndexQueryable();
 
         execute("INSERT INTO %s (id, v1) VALUES ('1', '0')");
         flush();
diff --git a/test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java b/test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java
index 82f0c512a0..dfb22ac74a 100644
--- a/test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/functional/CompactionTest.java
@@ -121,7 +121,6 @@ public class CompactionTest extends SAITester
     {
         createTable(CREATE_TABLE_TEMPLATE);
         String v1IndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
-        waitForIndexQueryable();
 
         int num = 10;
         for (int i = 0; i < num; i++)
@@ -249,7 +248,6 @@ public class CompactionTest extends SAITester
 
                             // build indexes on SSTables that will be compacted soon
                             createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
-                            waitForIndexQueryable();
 
                             // continue in-progress compaction
                             compactionLatch.countDown();
diff --git a/test/unit/org/apache/cassandra/index/sai/functional/DiskSpaceTest.java b/test/unit/org/apache/cassandra/index/sai/functional/DiskSpaceTest.java
index 254f4179a7..152d472b00 100644
--- a/test/unit/org/apache/cassandra/index/sai/functional/DiskSpaceTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/functional/DiskSpaceTest.java
@@ -41,7 +41,6 @@ public class DiskSpaceTest extends SAITester
 
         // create index, disk space should include index components
         String indexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
-        waitForIndexQueryable();
 
         long indexSize = indexDiskSpaceUse();
         long sstableSizeWithIndex = totalDiskSpaceUsed();
diff --git a/test/unit/org/apache/cassandra/index/sai/functional/DropTableTest.java b/test/unit/org/apache/cassandra/index/sai/functional/DropTableTest.java
index ae8c7670e6..191525e171 100644
--- a/test/unit/org/apache/cassandra/index/sai/functional/DropTableTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/functional/DropTableTest.java
@@ -50,7 +50,6 @@ public class DropTableTest extends SAITester
     {
         createTable(CREATE_TABLE_TEMPLATE);
         createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
-        waitForIndexQueryable();
 
         int rows = 100;
         for (int j = 0; j < rows; j++)
diff --git a/test/unit/org/apache/cassandra/index/sai/functional/FailureTest.java b/test/unit/org/apache/cassandra/index/sai/functional/FailureTest.java
index 3650e28621..893e304e65 100644
--- a/test/unit/org/apache/cassandra/index/sai/functional/FailureTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/functional/FailureTest.java
@@ -110,7 +110,7 @@ public class FailureTest extends SAITester
         Injection ssTableContextCreationFailure = newFailureOnEntry("context_failure_on_creation", SSTableContext.class, "<init>", RuntimeException.class);
         Injections.inject(ssTableContextCreationFailure);
 
-        String v1IndexName = createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
+        String v1IndexName = createIndexAsync(String.format(CREATE_INDEX_TEMPLATE, "v1"));
 
         // Verify that the initial index build fails...
         verifyInitialIndexFailed(v1IndexName);
diff --git a/test/unit/org/apache/cassandra/index/sai/functional/GroupComponentsTest.java b/test/unit/org/apache/cassandra/index/sai/functional/GroupComponentsTest.java
index a6d426e05e..b8049b7d0f 100644
--- a/test/unit/org/apache/cassandra/index/sai/functional/GroupComponentsTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/functional/GroupComponentsTest.java
@@ -46,7 +46,6 @@ public class GroupComponentsTest extends SAITester
     {
         createTable("CREATE TABLE %s (pk int primary key, value text)");
         createIndex("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex'");
-        waitForIndexQueryable();
         execute("INSERT INTO %s (pk) VALUES (1)");
         flush();
 
@@ -72,7 +71,6 @@ public class GroupComponentsTest extends SAITester
     {
         createTable("CREATE TABLE %s (pk int primary key, value text)");
         createIndex("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex'");
-        waitForIndexQueryable();
         execute("INSERT INTO %s (pk) VALUES (1)");
         flush();
 
@@ -94,7 +92,6 @@ public class GroupComponentsTest extends SAITester
     {
         createTable("CREATE TABLE %s (pk int primary key, value text)");
         IndexContext indexContext = createIndexContext(createIndex("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex'"), UTF8Type.instance);
-        waitForIndexQueryable();
         execute("INSERT INTO %s (pk, value) VALUES (1, '1')");
         flush();
 
diff --git a/test/unit/org/apache/cassandra/index/sai/functional/NodeRestartTest.java b/test/unit/org/apache/cassandra/index/sai/functional/NodeRestartTest.java
index 76461238fc..1766b69768 100644
--- a/test/unit/org/apache/cassandra/index/sai/functional/NodeRestartTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/functional/NodeRestartTest.java
@@ -30,8 +30,7 @@ import org.apache.cassandra.index.sai.StorageAttachedIndex;
 import org.apache.cassandra.inject.Injection;
 import org.apache.cassandra.inject.Injections;
 import org.apache.cassandra.inject.InvokePointBuilder;
-
-import static org.junit.Assert.assertFalse;
+import org.assertj.core.api.Assertions;
 
 public class NodeRestartTest extends SAITester
 {
@@ -65,7 +64,7 @@ public class NodeRestartTest extends SAITester
         // We should have completed no actual SSTable validations:
         assertValidationCount(0, 0);
 
-        assertFalse(isIndexQueryable());
+        Assertions.assertThat(getNotQueryableIndexes()).isNotEmpty();
     }
 
     // We don't allow the node to actually join the ring before a valid index is ready to accept queries.
@@ -113,7 +112,6 @@ public class NodeRestartTest extends SAITester
         flush();
 
         IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), UTF8Type.instance);
-        waitForIndexQueryable();
         verifyIndexFiles(literalIndexContext, 1);
         assertNumRows(1, "SELECT * FROM %%s WHERE v1 = '0'");
         assertValidationCount(0, 0);
@@ -124,7 +122,7 @@ public class NodeRestartTest extends SAITester
 
         assertNumRows(1, "SELECT * FROM %%s WHERE v1 = '0'");
 
-        waitForIndexQueryable();
+        waitForTableIndexesQueryable();
 
         // index components are included after restart
         verifyIndexComponentsIncludedInSSTable();
@@ -172,7 +170,6 @@ public class NodeRestartTest extends SAITester
         flush();
 
         IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), UTF8Type.instance);
-        waitForIndexQueryable();
         verifyIndexFiles(literalIndexContext, 1);
         assertNumRows(1, "SELECT * FROM %%s WHERE v1 = '0'");
         assertValidationCount(0, 0);
diff --git a/test/unit/org/apache/cassandra/index/sai/functional/SnapshotTest.java b/test/unit/org/apache/cassandra/index/sai/functional/SnapshotTest.java
index f0c78986bc..769435d309 100644
--- a/test/unit/org/apache/cassandra/index/sai/functional/SnapshotTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/functional/SnapshotTest.java
@@ -52,7 +52,6 @@ public class SnapshotTest extends SAITester
         // Insert some initial data and create the index over it
         execute("INSERT INTO %s (id1, v1) VALUES ('0', '0');");
         IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), UTF8Type.instance);
-        waitForIndexQueryable();
         flush();
         verifyIndexFiles(literalIndexContext, 1, 1, 1);
         // Note: This test will fail here if it is run on its own because the per-index validation
@@ -130,7 +129,6 @@ public class SnapshotTest extends SAITester
 
         // create index
         IndexContext literalIndexContext = createIndexContext(createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1")), UTF8Type.instance);
-        waitForIndexQueryable();
         verifyIndexFiles(literalIndexContext, 2);
         assertValidationCount(0, 0);
 
diff --git a/test/unit/org/apache/cassandra/index/sai/metrics/QueryMetricsTest.java b/test/unit/org/apache/cassandra/index/sai/metrics/QueryMetricsTest.java
index ade72d9f18..14931b173b 100644
--- a/test/unit/org/apache/cassandra/index/sai/metrics/QueryMetricsTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/metrics/QueryMetricsTest.java
@@ -83,7 +83,6 @@ public class QueryMetricsTest extends AbstractMetricsTest
 
         createTable(String.format(CREATE_TABLE_TEMPLATE, keyspace, table));
         createIndex(String.format(CREATE_INDEX_TEMPLATE, index, keyspace, table, "v1"));
-        waitForIndexQueryable(keyspace, table);
 
         execute("INSERT INTO " + keyspace + '.' + table + " (id1, v1, v2) VALUES ('0', 0, '0')");
 
diff --git a/test/unit/org/apache/cassandra/index/sai/metrics/SegmentFlushingFailureTester.java b/test/unit/org/apache/cassandra/index/sai/metrics/SegmentFlushingFailureTester.java
index 98ae5a4d3e..6bc6c996fa 100644
--- a/test/unit/org/apache/cassandra/index/sai/metrics/SegmentFlushingFailureTester.java
+++ b/test/unit/org/apache/cassandra/index/sai/metrics/SegmentFlushingFailureTester.java
@@ -169,7 +169,6 @@ public abstract class SegmentFlushingFailureTester extends SAITester
         createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
         String table2 = createTable(CREATE_TABLE_TEMPLATE);
         createIndex(String.format(CREATE_INDEX_TEMPLATE, "v1"));
-        waitForIndexQueryable();
 
         assertEquals(expectedBytesLimit(), getSegmentBufferSpaceLimit());
         assertEquals("Segment buffer memory tracker should start at zero!", 0L, getSegmentBufferUsedBytes());
diff --git a/test/unit/org/apache/cassandra/index/sai/metrics/StateMetricsTest.java b/test/unit/org/apache/cassandra/index/sai/metrics/StateMetricsTest.java
index fc5e41bce6..e6ecda02fb 100644
--- a/test/unit/org/apache/cassandra/index/sai/metrics/StateMetricsTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/metrics/StateMetricsTest.java
@@ -48,7 +48,6 @@ public class StateMetricsTest extends AbstractMetricsTest
 
         createTable(String.format(CREATE_TABLE_TEMPLATE, keyspace, table));
         createIndex(String.format(CREATE_INDEX_TEMPLATE, index, keyspace, table, "v1"));
-        waitForIndexQueryable(keyspace, table);
 
         execute("INSERT INTO " + keyspace + '.' + table + " (id1, v1, v2) VALUES ('0', 0, '0')");
 
@@ -73,9 +72,8 @@ public class StateMetricsTest extends AbstractMetricsTest
 
         String keyspace = createKeyspace(CREATE_KEYSPACE_TEMPLATE);
         createTable(String.format(CREATE_TABLE_TEMPLATE, keyspace, table));
-        createIndex(String.format(CREATE_INDEX_TEMPLATE, index+"_v1", keyspace, table, "v1"));
-        createIndex(String.format(CREATE_INDEX_TEMPLATE, index+"_v2", keyspace, table, "v2"));
-        waitForIndexQueryable(keyspace, table);
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, index + "_v1", keyspace, table, "v1"));
+        createIndex(String.format(CREATE_INDEX_TEMPLATE, index + "_v2", keyspace, table, "v2"));
 
         execute("INSERT INTO " + keyspace + '.' + table + " (id1, v1, v2) VALUES ('0', 0, '0')");
         execute("INSERT INTO " + keyspace + '.' + table + " (id1, v1, v2) VALUES ('1', 1, '1')");
diff --git a/test/unit/org/apache/cassandra/index/sai/plan/OperationTest.java b/test/unit/org/apache/cassandra/index/sai/plan/OperationTest.java
index f957ddd3f2..10ed7f5035 100644
--- a/test/unit/org/apache/cassandra/index/sai/plan/OperationTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/plan/OperationTest.java
@@ -78,7 +78,7 @@ public class OperationTest
     private static final String KS_NAME = "sai";
     private static final String CF_NAME = "test_cf";
     private static final String CLUSTERING_CF_NAME = "clustering_test_cf";
-    private static final String STATIC_CF_NAME = "static_ndi_test_cf";
+    private static final String STATIC_CF_NAME = "static_sai_test_cf";
 
     private static ColumnFamilyStore BACKEND;
     private static ColumnFamilyStore CLUSTERING_BACKEND;
diff --git a/test/unit/org/apache/cassandra/index/sai/virtual/IndexesSystemViewTest.java b/test/unit/org/apache/cassandra/index/sai/virtual/IndexesSystemViewTest.java
index aabbb13078..a8f5bf9a58 100644
--- a/test/unit/org/apache/cassandra/index/sai/virtual/IndexesSystemViewTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/virtual/IndexesSystemViewTest.java
@@ -73,14 +73,14 @@ public class IndexesSystemViewTest extends SAITester
 
         // create the index simulating a long build and verify that there is an empty record in the virtual table
         Injections.inject(blockIndexBuild);
-        String v1IndexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName()));
+        String v1IndexName = createIndexAsync(String.format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", StorageAttachedIndex.class.getName()));
 
         assertRows(execute(SELECT), row(v1IndexName, "v1", false, true, true));
 
         // unblock the long build and verify that there is a finished empty record in the virtual table
         blockIndexBuild.countDown();
         blockIndexBuild.disable();
-        waitForIndexQueryable();
+        waitForTableIndexesQueryable();
         assertRows(execute(SELECT), row(v1IndexName, "v1", true, false, true));
 
         // insert some data and verify that virtual table record is still empty since we haven't flushed yet
diff --git a/test/unit/org/apache/cassandra/index/sai/virtual/SSTablesSystemViewTest.java b/test/unit/org/apache/cassandra/index/sai/virtual/SSTablesSystemViewTest.java
index f35de185bd..eebcdf6561 100644
--- a/test/unit/org/apache/cassandra/index/sai/virtual/SSTablesSystemViewTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/virtual/SSTablesSystemViewTest.java
@@ -74,7 +74,6 @@ public class SSTablesSystemViewTest extends SAITester
     {
         createTable("CREATE TABLE %s (k text, c text, v1 text, v2 text, PRIMARY KEY (k, c))");
         String v1IndexName = createIndex("CREATE CUSTOM INDEX ON %s(v1) USING 'StorageAttachedIndex'");
-        waitForIndexQueryable();
 
         String insert = "INSERT INTO %s(k, c, v1, v2) VALUES (?, ?, ?, ?)";
 
@@ -101,7 +100,6 @@ public class SSTablesSystemViewTest extends SAITester
 
         // create a second index, this should create a new additional entry in the table for each sstable
         String v2IndexName = createIndex("CREATE CUSTOM INDEX ON %s(v2) USING 'StorageAttachedIndex'");
-        waitForIndexQueryable();
         Object[] row3 = readRow(v2IndexName, id1, "v2", 1L, 0L, 0L);
         Object[] row4 = readRow(v2IndexName, id2, "v2", 2L, 0L, 1L);
         assertRowsIgnoringOrder(execute(SELECT), row1, row2, row3, row4);
diff --git a/test/unit/org/apache/cassandra/index/sai/virtual/SegmentsSystemViewTest.java b/test/unit/org/apache/cassandra/index/sai/virtual/SegmentsSystemViewTest.java
index fd6046ee49..f37dcf86f0 100644
--- a/test/unit/org/apache/cassandra/index/sai/virtual/SegmentsSystemViewTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/virtual/SegmentsSystemViewTest.java
@@ -92,7 +92,6 @@ public class SegmentsSystemViewTest extends SAITester
     {
         createTable("CREATE TABLE %s (k int, c int, v1 text, PRIMARY KEY (k, c))");
         String literalIndex = createIndex("CREATE CUSTOM INDEX ON %s(v1) USING 'StorageAttachedIndex'");
-        waitForIndexQueryable();
 
         int num = 100;
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


[cassandra] 02/06: In-memory index implementation with query path

Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

maedhroz pushed a commit to branch cep-7-sai
in repository https://gitbox.apache.org/repos/asf/cassandra.git

commit 14ed6a7d73f7daaebe3a88865f8b672cd3b6d001
Author: Mike Adamson <ma...@datastax.com>
AuthorDate: Thu Jan 19 14:24:46 2023 +0000

    In-memory index implementation with query path
    
    This includes the following elements of the Storage Attached Index:
    - Memtable-attached indexes backed by an in-memory trie structure for byte-comparable values
    - Query path for the in-memory index
    - Index status propagation
    - Randomized testing for Memtable-attached indexes
    
    patch my Mike Adamson; reviewed by Caleb Rackliffe and Andres de la Peña for CASSANDRA-18058
    
    Co-authored-by: Mike Adamson <ma...@datastax.com>
    Co-authored-by: Caleb Rackliffe <ca...@gmail.com>
---
 .build/cassandra-build-deps-template.xml           |   4 +
 .build/cassandra-deps-template.xml                 |   4 +
 .build/parent-pom-template.xml                     |  17 +
 .../config/CassandraRelevantProperties.java        |   5 +
 .../cql3/restrictions/StatementRestrictions.java   |   2 +-
 .../cassandra/cql3/statements/CQL3CasRequest.java  |   2 +-
 .../cql3/statements/ModificationStatement.java     |   2 +-
 .../statements/schema/CreateIndexStatement.java    |  14 +-
 .../cassandra/db/PartitionRangeReadCommand.java    |   2 +-
 src/java/org/apache/cassandra/db/ReadQuery.java    |   2 +-
 .../cassandra/db/SinglePartitionReadCommand.java   |   6 +-
 .../cassandra/db/SinglePartitionReadQuery.java     |   2 +-
 .../org/apache/cassandra/db/filter/RowFilter.java  |  17 +-
 .../org/apache/cassandra/db/view/TableViews.java   |   2 +-
 .../cassandra/db/virtual/SystemViewsKeyspace.java  |   3 +
 .../apache/cassandra/dht/Murmur3Partitioner.java   |   6 +
 src/java/org/apache/cassandra/dht/Token.java       |  15 +
 .../cassandra/exceptions/RequestFailureReason.java |   3 +-
 .../org/apache/cassandra/gms/ApplicationState.java |   1 +
 .../org/apache/cassandra/gms/VersionedValue.java   |   5 +
 src/java/org/apache/cassandra/index/Index.java     |  14 +-
 .../apache/cassandra/index/IndexStatusManager.java | 242 +++++++
 .../cassandra/index/SecondaryIndexManager.java     |  31 +-
 .../internal/composites/CompositesSearcher.java    |   2 +-
 .../apache/cassandra/index/sai/IndexContext.java   | 453 +++++++++++++
 .../apache/cassandra/index/sai/QueryContext.java   |  68 ++
 src/java/org/apache/cassandra/index/sai/README.md  | 115 ++++
 .../cassandra/index/sai/StorageAttachedIndex.java  | 449 +++++++++++++
 .../index/sai/StorageAttachedIndexGroup.java       | 237 +++++++
 .../index/sai/analyzer/AbstractAnalyzer.java       |  87 +++
 .../cassandra/index/sai/analyzer/NoOpAnalyzer.java |  66 ++
 .../memory/FilteringInMemoryKeyRangeIterator.java  |  47 ++
 .../index/sai/memory/InMemoryKeyRangeIterator.java | 101 +++
 .../cassandra/index/sai/memory/MemtableIndex.java  |  92 +++
 .../index/sai/memory/TrieMemoryIndex.java          | 383 +++++++++++
 .../index/sai/metrics/AbstractMetrics.java         |  91 +++
 .../cassandra/index/sai/metrics/IndexMetrics.java  |  40 ++
 .../index/sai/metrics/TableQueryMetrics.java       | 115 ++++
 .../index/sai/metrics/TableStateMetrics.java       |  39 ++
 .../cassandra/index/sai/plan/Expression.java       | 378 +++++++++++
 .../cassandra/index/sai/plan/FilterTree.java       | 138 ++++
 .../apache/cassandra/index/sai/plan/Operation.java | 351 +++++++++++
 .../cassandra/index/sai/plan/QueryController.java  | 246 ++++++++
 .../sai/plan/StorageAttachedIndexQueryPlan.java    | 135 ++++
 .../sai/plan/StorageAttachedIndexSearcher.java     | 566 +++++++++++++++++
 .../index/sai/utils/KeyRangeConcatIterator.java    | 181 ++++++
 .../sai/utils/KeyRangeIntersectionIterator.java    | 286 +++++++++
 .../index/sai/utils/KeyRangeIterator.java          | 210 +++++++
 .../index/sai/utils/KeyRangeUnionIterator.java     | 171 +++++
 .../cassandra/index/sai/utils/PrimaryKey.java      | 187 ++++++
 .../index/sai/utils/PrimaryKeyFactory.java         |  57 ++
 .../cassandra/index/sai/utils/PrimaryKeys.java     |  75 +++
 .../apache/cassandra/index/sai/utils/TypeUtil.java | 472 ++++++++++++++
 .../index/sai/virtual/IndexesSystemView.java       | 115 ++++
 .../sai/virtual/StorageAttachedIndexTables.java    |  35 ++
 .../cassandra/index/sasi/disk/OnDiskIndex.java     |   6 +-
 .../cassandra/index/sasi/disk/TokenTree.java       |   4 +-
 .../index/sasi/memory/KeyRangeIterator.java        |   5 +-
 .../cassandra/index/sasi/utils/RangeIterator.java  |   4 +-
 .../org/apache/cassandra/locator/ReplicaPlans.java |  32 +-
 .../org/apache/cassandra/schema/IndexMetadata.java |  19 +-
 .../apache/cassandra/service/StorageService.java   |  20 +-
 .../service/reads/AbstractReadExecutor.java        |  10 +-
 .../service/reads/ReplicaFilteringProtection.java  |   2 +-
 .../service/reads/range/RangeCommands.java         |  11 +-
 .../service/reads/range/ReplicaPlanIterator.java   |  12 +-
 .../AbstractGuavaIterator.java}                    |  28 +-
 .../org/apache/cassandra/utils/ByteBufferUtil.java |   9 +
 .../test/sai/IndexAvailabilityTest.java            | 288 +++++++++
 .../cassandra/distributed/test/sai/SAIUtil.java    | 141 +++++
 test/unit/org/apache/cassandra/cql3/CQLTester.java |  15 +
 .../unit/org/apache/cassandra/db/KeyspaceTest.java |  16 +-
 .../cassandra/db/ReadCommandVerbHandlerTest.java   |   2 +-
 .../org/apache/cassandra/db/ReadResponseTest.java  |   2 +-
 .../db/SinglePartitionSliceCommandTest.java        |   6 +-
 .../cassandra/db/compaction/CompactionsTest.java   |   2 +-
 .../apache/cassandra/db/filter/RowFilterTest.java  |   4 +-
 .../UnfilteredRowIteratorWithLowerBoundTest.java   |   4 +-
 .../cassandra/db/virtual/GossipInfoTableTest.java  |   2 +-
 .../org/apache/cassandra/dht/KeyCollisionTest.java |   6 +
 .../cassandra/index/SecondaryIndexManagerTest.java |   1 -
 .../cassandra/index/sai/IndexingSchemaLoader.java  | 173 +++++
 .../org/apache/cassandra/index/sai/SAITester.java  | 360 +++++++++++
 .../index/sai/cql/AbstractQueryTester.java         |  78 +++
 .../cassandra/index/sai/cql/BaseDataModel.java     | 397 ++++++++++++
 .../cassandra/index/sai/cql/BooleanTypeTest.java   |  44 ++
 .../index/sai/cql/ClusteringKeyIndexTest.java      |  76 +++
 .../index/sai/cql/CollectionIndexingTest.java      | 233 +++++++
 .../sai/cql/CompositePartitionKeyIndexTest.java    | 104 +++
 .../index/sai/cql/DecimalLargeValueTest.java       | 151 +++++
 .../index/sai/cql/DuplicateRowIDTest.java          |  63 ++
 .../cassandra/index/sai/cql/IndexQuerySupport.java | 482 ++++++++++++++
 .../sai/cql/InetAddressTypeEquivalencyTest.java    | 200 ++++++
 .../sai/cql/MixedIndexImplementationsTest.java     | 190 ++++++
 .../index/sai/cql/MultipleColumnIndexTest.java     |  63 ++
 .../index/sai/cql/QueryTimeToLiveTest.java         |  29 +
 .../index/sai/cql/QueryWriteLifecycleTest.java     |  29 +
 .../index/sai/cql/SingleNodeExecutor.java          |  76 +++
 .../index/sai/cql/StorageAttachedIndexDDLTest.java | 320 ++++++++++
 .../cassandra/index/sai/cql/types/AsciiTest.java   |  31 +
 .../cassandra/index/sai/cql/types/BigintTest.java  |  31 +
 .../cassandra/index/sai/cql/types/BooleanTest.java |  32 +
 .../cassandra/index/sai/cql/types/DataSet.java     | 697 +++++++++++++++++++++
 .../cassandra/index/sai/cql/types/DateTest.java    |  31 +
 .../cassandra/index/sai/cql/types/DecimalTest.java |  31 +
 .../cassandra/index/sai/cql/types/DoubleTest.java  |  31 +
 .../cassandra/index/sai/cql/types/FloatTest.java   |  31 +
 .../index/sai/cql/types/IndexingTypeSupport.java   | 123 ++++
 .../cassandra/index/sai/cql/types/InetTest.java    |  31 +
 .../cassandra/index/sai/cql/types/IntTest.java     |  31 +
 .../sai/cql/types/NumericTypeSortingTest.java      |  96 +++
 .../cassandra/index/sai/cql/types/QuerySet.java    | 536 ++++++++++++++++
 .../index/sai/cql/types/SmallintTest.java          |  31 +
 .../cassandra/index/sai/cql/types/TextTest.java    |  31 +
 .../cassandra/index/sai/cql/types/TimeTest.java    |  31 +
 .../index/sai/cql/types/TimestampTest.java         |  31 +
 .../index/sai/cql/types/TimeuuidTest.java          |  31 +
 .../cassandra/index/sai/cql/types/TinyintTest.java |  31 +
 .../cassandra/index/sai/cql/types/UuidTest.java    |  31 +
 .../cassandra/index/sai/cql/types/VarintTest.java  |  31 +
 .../cql/types/collections/CollectionDataSet.java   | 284 +++++++++
 .../collections/lists/FrozenListAsciiTest.java     |  35 ++
 .../collections/lists/FrozenListDecimalTest.java   |  35 ++
 .../types/collections/lists/FrozenListIntTest.java |  35 ++
 .../collections/lists/FrozenListVarintTest.java    |  35 ++
 .../cql/types/collections/lists/ListAsciiTest.java |  35 ++
 .../types/collections/lists/ListBigintTest.java    |  35 ++
 .../cql/types/collections/lists/ListDateTest.java  |  35 ++
 .../types/collections/lists/ListDecimalTest.java   |  35 ++
 .../types/collections/lists/ListDoubleTest.java    |  35 ++
 .../cql/types/collections/lists/ListFloatTest.java |  35 ++
 .../lists/ListFrozenCollectionTest.java            |  37 ++
 .../cql/types/collections/lists/ListInetTest.java  |  35 ++
 .../cql/types/collections/lists/ListIntTest.java   |  35 ++
 .../types/collections/lists/ListSmallintTest.java  |  35 ++
 .../cql/types/collections/lists/ListTextTest.java  |  35 ++
 .../cql/types/collections/lists/ListTimeTest.java  |  35 ++
 .../types/collections/lists/ListTimestampTest.java |  35 ++
 .../types/collections/lists/ListTimeuuidTest.java  |  35 ++
 .../types/collections/lists/ListTinyintTest.java   |  35 ++
 .../cql/types/collections/lists/ListUuidTest.java  |  35 ++
 .../types/collections/lists/ListVarintTest.java    |  35 ++
 .../types/collections/maps/FrozenMapAsciiTest.java |  35 ++
 .../collections/maps/FrozenMapDecimalTest.java     |  35 ++
 .../types/collections/maps/FrozenMapIntTest.java   |  35 ++
 .../collections/maps/FrozenMapVarintTest.java      |  35 ++
 .../cql/types/collections/maps/MapAsciiTest.java   |  35 ++
 .../cql/types/collections/maps/MapBigintTest.java  |  35 ++
 .../cql/types/collections/maps/MapDateTest.java    |  35 ++
 .../cql/types/collections/maps/MapDecimalTest.java |  36 ++
 .../cql/types/collections/maps/MapDoubleTest.java  |  35 ++
 .../collections/maps/MapEntriesAsciiTest.java      |  35 ++
 .../maps/MapEntriesFrozenCollectionTest.java       |  37 ++
 .../types/collections/maps/MapEntriesIntTest.java  |  35 ++
 .../collections/maps/MapEntriesVarintTest.java     |  35 ++
 .../cql/types/collections/maps/MapFloatTest.java   |  35 ++
 .../collections/maps/MapFrozenCollectionTest.java  |  37 ++
 .../cql/types/collections/maps/MapInetTest.java    |  35 ++
 .../sai/cql/types/collections/maps/MapIntTest.java |  35 ++
 .../types/collections/maps/MapKeysAsciiTest.java   |  35 ++
 .../maps/MapKeysFrozenCollectionTest.java          |  37 ++
 .../cql/types/collections/maps/MapKeysIntTest.java |  35 ++
 .../types/collections/maps/MapKeysVarintTest.java  |  35 ++
 .../types/collections/maps/MapSmallintTest.java    |  35 ++
 .../cql/types/collections/maps/MapTextTest.java    |  35 ++
 .../cql/types/collections/maps/MapTimeTest.java    |  35 ++
 .../types/collections/maps/MapTimestampTest.java   |  35 ++
 .../types/collections/maps/MapTimeuuidTest.java    |  35 ++
 .../cql/types/collections/maps/MapTinyintTest.java |  35 ++
 .../cql/types/collections/maps/MapUuidTest.java    |  35 ++
 .../types/collections/maps/MapValuesAsciiTest.java |  35 ++
 .../maps/MapValuesFrozenCollectionTest.java        |  37 ++
 .../types/collections/maps/MapValuesIntTest.java   |  35 ++
 .../collections/maps/MapValuesVarintTest.java      |  35 ++
 .../cql/types/collections/maps/MapVarintTest.java  |  35 ++
 .../types/collections/maps/MultiMapAsciiTest.java  |  35 ++
 .../types/collections/maps/MultiMapIntTest.java    |  35 ++
 .../types/collections/maps/MultiMapVarintTest.java |  35 ++
 .../types/collections/sets/FrozenSetAsciiTest.java |  35 ++
 .../collections/sets/FrozenSetDecimalTest.java     |  35 ++
 .../types/collections/sets/FrozenSetIntTest.java   |  35 ++
 .../collections/sets/FrozenSetVarintTest.java      |  35 ++
 .../cql/types/collections/sets/SetAsciiTest.java   |  35 ++
 .../cql/types/collections/sets/SetBigintTest.java  |  35 ++
 .../cql/types/collections/sets/SetDateTest.java    |  35 ++
 .../cql/types/collections/sets/SetDecimalTest.java |  35 ++
 .../cql/types/collections/sets/SetDoubleTest.java  |  35 ++
 .../cql/types/collections/sets/SetFloatTest.java   |  35 ++
 .../collections/sets/SetFrozenCollectionTest.java  |  37 ++
 .../cql/types/collections/sets/SetInetTest.java    |  35 ++
 .../sai/cql/types/collections/sets/SetIntTest.java |  35 ++
 .../types/collections/sets/SetSmallintTest.java    |  35 ++
 .../cql/types/collections/sets/SetTextTest.java    |  35 ++
 .../cql/types/collections/sets/SetTimeTest.java    |  35 ++
 .../types/collections/sets/SetTimestampTest.java   |  35 ++
 .../types/collections/sets/SetTimeuuidTest.java    |  35 ++
 .../cql/types/collections/sets/SetTinyintTest.java |  35 ++
 .../cql/types/collections/sets/SetUuidTest.java    |  35 ++
 .../cql/types/collections/sets/SetVarintTest.java  |  35 ++
 .../types/multicell/FrozenTupleCollectionTest.java |  40 ++
 .../cql/types/multicell/FrozenTupleDataSet.java    |  74 +++
 .../sai/cql/types/multicell/FrozenTupleTest.java   |  35 ++
 .../cql/types/multicell/FrozenTupleTupleTest.java  |  35 ++
 .../types/multicell/FrozenUDTCollectionTest.java   |  38 ++
 .../sai/cql/types/multicell/FrozenUDTDataSet.java  |  83 +++
 .../sai/cql/types/multicell/FrozenUDTTest.java     |  35 ++
 .../sai/cql/types/multicell/TupleDataSet.java      |  36 ++
 .../index/sai/cql/types/multicell/TupleTest.java   |  35 ++
 .../AbstractInMemoryKeyRangeIteratorTester.java    | 174 +++++
 .../sai/memory/InMemoryKeyRangeIteratorTest.java   |  38 ++
 .../PriorityInMemoryKeyRangeIteratorTest.java      |  40 ++
 .../index/sai/memory/TrieMemoryIndexTest.java      | 254 ++++++++
 .../index/sai/metrics/AbstractMetricsTest.java     |  37 ++
 .../index/sai/metrics/IndexMetricsTest.java        |  75 +++
 .../index/sai/metrics/QueryMetricsTest.java        | 108 ++++
 .../index/sai/metrics/StateMetricsTest.java        |  98 +++
 .../cassandra/index/sai/plan/ExpressionTest.java   |  54 ++
 .../cassandra/index/sai/plan/OperationTest.java    | 471 ++++++++++++++
 .../sai/utils/AbstractKeyRangeIteratorTester.java  | 130 ++++
 .../index/sai/utils/AbstractPrimaryKeyTester.java  | 171 +++++
 .../sai/utils/KeyRangeConcatIteratorTest.java      | 420 +++++++++++++
 .../utils/KeyRangeIntersectionIteratorTest.java    | 436 +++++++++++++
 .../index/sai/utils/KeyRangeUnionIteratorTest.java | 445 +++++++++++++
 .../cassandra/index/sai/utils/LongIterator.java    | 117 ++++
 .../cassandra/index/sai/utils/PrimaryKeyTest.java  | 381 +++++++++++
 .../index/sai/utils/ResourceLeakDetector.java      |  81 +++
 .../index/sai/utils/SAIRandomizedTester.java       |  98 +++
 .../index/sai/virtual/IndexesSystemViewTest.java   | 156 +++++
 .../apache/cassandra/index/sasi/SASIIndexTest.java |   2 +-
 .../org/apache/cassandra/inject/ActionBuilder.java | 294 +++++++++
 .../org/apache/cassandra/inject/CyclicBarrier.java |  71 +++
 .../org/apache/cassandra/inject/Expression.java    | 105 ++++
 .../org/apache/cassandra/inject/Injection.java     |  87 +++
 .../apache/cassandra/inject/InjectionBuilder.java  |  42 ++
 .../org/apache/cassandra/inject/Injections.java    | 510 +++++++++++++++
 .../cassandra/inject/InvokePointBuilder.java       | 133 ++++
 test/unit/org/apache/cassandra/inject/Rule.java    |  59 ++
 .../locator/AssureSufficientLiveNodesTest.java     |  10 +-
 .../apache/cassandra/service/QueryPagerTest.java   |   2 +-
 .../reads/range/RangeCommandIteratorTest.java      |   4 +-
 .../service/reads/range/RangeCommandsTest.java     |   6 +-
 .../reads/range/ReplicaPlanIteratorTest.java       |   2 +-
 .../service/reads/range/ReplicaPlanMergerTest.java |   2 +-
 .../reads/repair/RepairedDataVerifierTest.java     |   2 +-
 244 files changed, 20533 insertions(+), 108 deletions(-)

diff --git a/.build/cassandra-build-deps-template.xml b/.build/cassandra-build-deps-template.xml
index cbf2f8c59d..39c338478d 100644
--- a/.build/cassandra-build-deps-template.xml
+++ b/.build/cassandra-build-deps-template.xml
@@ -123,5 +123,9 @@
       <groupId>com.fasterxml.jackson.dataformat</groupId>
       <artifactId>jackson-dataformat-yaml</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.carrotsearch.randomizedtesting</groupId>
+      <artifactId>randomizedtesting-runner</artifactId>
+    </dependency>
   </dependencies>
 </project>
diff --git a/.build/cassandra-deps-template.xml b/.build/cassandra-deps-template.xml
index e907bffa82..c2194a0ab7 100644
--- a/.build/cassandra-deps-template.xml
+++ b/.build/cassandra-deps-template.xml
@@ -376,5 +376,9 @@
       <groupId>org.agrona</groupId>
       <artifactId>agrona</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-core</artifactId>
+    </dependency>
   </dependencies>
 </project>
diff --git a/.build/parent-pom-template.xml b/.build/parent-pom-template.xml
index e4b63bf3a8..19f767a278 100644
--- a/.build/parent-pom-template.xml
+++ b/.build/parent-pom-template.xml
@@ -1042,6 +1042,23 @@
         <artifactId>agrona</artifactId>
         <version>1.17.1</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.lucene</groupId>
+        <artifactId>lucene-core</artifactId>
+        <version>7.5.0</version>
+      </dependency>
+      <dependency>
+        <groupId>com.carrotsearch.randomizedtesting</groupId>
+        <artifactId>randomizedtesting-runner</artifactId>
+        <version>2.1.2</version>
+        <scope>test</scope>
+        <exclusions>
+          <exclusion>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
       <dependency>
         <groupId>ch.obermuhlner</groupId>
         <artifactId>big-math</artifactId>
diff --git a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
index 4daa825665..c3fdb8aed2 100644
--- a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
+++ b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
@@ -393,6 +393,11 @@ public enum CassandraRelevantProperties
     SCHEMA_PULL_INTERVAL_MS("cassandra.schema_pull_interval_ms", "60000"),
     SCHEMA_UPDATE_HANDLER_FACTORY_CLASS("cassandra.schema.update_handler_factory.class"),
     SEARCH_CONCURRENCY_FACTOR("cassandra.search_concurrency_factor", "1"),
+
+    // SAI specific properties
+    /** Controls the maximum number of index query intersections that will take part in a query */
+    SAI_INTERSECTION_CLAUSE_LIMIT("cassandra.sai.intersection.clause.limit", "2"),
+
     /**
      * The maximum number of seeds returned by a seed provider before emmitting a warning.
      * A large seed list may impact effectiveness of the third gossip round.
diff --git a/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java b/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
index d522f46d2f..52deaabbfa 100644
--- a/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
+++ b/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java
@@ -678,7 +678,7 @@ public final class StatementRestrictions
     public RowFilter getRowFilter(IndexRegistry indexRegistry, QueryOptions options)
     {
         if (filterRestrictions.isEmpty())
-            return RowFilter.NONE;
+            return RowFilter.none();
 
         RowFilter filter = RowFilter.create();
         for (Restrictions restrictions : filterRestrictions.getRestrictions())
diff --git a/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java b/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java
index 3db4793c56..b68800a8fc 100644
--- a/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java
+++ b/src/java/org/apache/cassandra/cql3/statements/CQL3CasRequest.java
@@ -194,7 +194,7 @@ public class CQL3CasRequest implements CASRequest
             return SinglePartitionReadCommand.create(metadata,
                                                    nowInSec,
                                                    columnFilter,
-                                                   RowFilter.NONE,
+                                                   RowFilter.none(),
                                                    DataLimits.cqlLimits(1),
                                                    key,
                                                    new ClusteringIndexSliceFilter(Slices.ALL, false));
diff --git a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java
index 0cf677187c..dec057c70a 100644
--- a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java
@@ -442,7 +442,7 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa
             commands.add(SinglePartitionReadCommand.create(metadata(),
                                                            nowInSeconds,
                                                            ColumnFilter.selection(this.requiresRead),
-                                                           RowFilter.NONE,
+                                                           RowFilter.none(),
                                                            limits,
                                                            metadata().partitioner.decorateKey(key),
                                                            filter));
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java
index 7a3a41e618..73a7b6aac6 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java
@@ -48,6 +48,9 @@ import static com.google.common.collect.Iterables.tryFind;
 
 public final class CreateIndexStatement extends AlterSchemaStatement
 {
+    public static final String INVALID_CUSTOM_INDEX_TARGET = "Column '%s' is longer than the permissible name length of %d characters or" +
+                                                             " contains non-alphanumeric-underscore characters";
+
     private final String indexName;
     private final String tableName;
     private final List<IndexTarget.Raw> rawIndexTargets;
@@ -138,11 +141,11 @@ public final class CreateIndexStatement extends AlterSchemaStatement
                     throw ire("Duplicate column '%s' in index target list", target.column);
         }
 
-        indexTargets.forEach(t -> validateIndexTarget(table, t));
+        IndexMetadata.Kind kind = attrs.isCustom ? IndexMetadata.Kind.CUSTOM : IndexMetadata.Kind.COMPOSITES;
 
-        String name = null == indexName ? generateIndexName(keyspace, indexTargets) : indexName;
+        indexTargets.forEach(t -> validateIndexTarget(table, kind, t));
 
-        IndexMetadata.Kind kind = attrs.isCustom ? IndexMetadata.Kind.CUSTOM : IndexMetadata.Kind.COMPOSITES;
+        String name = null == indexName ? generateIndexName(keyspace, indexTargets) : indexName;
 
         Map<String, String> options = attrs.isCustom ? attrs.getOptions() : Collections.emptyMap();
 
@@ -173,13 +176,16 @@ public final class CreateIndexStatement extends AlterSchemaStatement
         return ImmutableSet.of();
     }
 
-    private void validateIndexTarget(TableMetadata table, IndexTarget target)
+    private void validateIndexTarget(TableMetadata table, IndexMetadata.Kind kind, IndexTarget target)
     {
         ColumnMetadata column = table.getColumn(target.column);
 
         if (null == column)
             throw ire("Column '%s' doesn't exist", target.column);
 
+        if ((kind == IndexMetadata.Kind.CUSTOM) && !SchemaConstants.isValidName(target.column.toString()))
+            throw ire(INVALID_CUSTOM_INDEX_TARGET, target.column, SchemaConstants.NAME_LENGTH);
+
         if (column.type.referencesDuration())
         {
             if (column.type.isCollection())
diff --git a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
index 4a47a96189..37227971bc 100644
--- a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
+++ b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
@@ -158,7 +158,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                       metadata,
                       nowInSec,
                       ColumnFilter.all(metadata),
-                      RowFilter.NONE,
+                      RowFilter.none(),
                       DataLimits.NONE,
                       DataRange.allData(metadata.partitioner),
                       null,
diff --git a/src/java/org/apache/cassandra/db/ReadQuery.java b/src/java/org/apache/cassandra/db/ReadQuery.java
index 55a2cf6969..a175ca5cae 100644
--- a/src/java/org/apache/cassandra/db/ReadQuery.java
+++ b/src/java/org/apache/cassandra/db/ReadQuery.java
@@ -107,7 +107,7 @@ public interface ReadQuery
             @Override
             public RowFilter rowFilter()
             {
-                return RowFilter.NONE;
+                return RowFilter.none();
             }
 
             @Override
diff --git a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
index ba97e12b5b..7901c99421 100644
--- a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
+++ b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
@@ -237,7 +237,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                                     ColumnFilter columnFilter,
                                                     ClusteringIndexFilter filter)
     {
-        return create(metadata, nowInSec, columnFilter, RowFilter.NONE, DataLimits.NONE, key, filter);
+        return create(metadata, nowInSec, columnFilter, RowFilter.none(), DataLimits.NONE, key, filter);
     }
 
     /**
@@ -298,7 +298,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
     public static SinglePartitionReadCommand create(TableMetadata metadata, int nowInSec, DecoratedKey key, Slices slices)
     {
         ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(slices, false);
-        return create(metadata, nowInSec, ColumnFilter.all(metadata), RowFilter.NONE, DataLimits.NONE, key, filter);
+        return create(metadata, nowInSec, ColumnFilter.all(metadata), RowFilter.none(), DataLimits.NONE, key, filter);
     }
 
     /**
@@ -331,7 +331,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
     public static SinglePartitionReadCommand create(TableMetadata metadata, int nowInSec, DecoratedKey key, NavigableSet<Clustering<?>> names)
     {
         ClusteringIndexNamesFilter filter = new ClusteringIndexNamesFilter(names, false);
-        return create(metadata, nowInSec, ColumnFilter.all(metadata), RowFilter.NONE, DataLimits.NONE, key, filter);
+        return create(metadata, nowInSec, ColumnFilter.all(metadata), RowFilter.none(), DataLimits.NONE, key, filter);
     }
 
     /**
diff --git a/src/java/org/apache/cassandra/db/SinglePartitionReadQuery.java b/src/java/org/apache/cassandra/db/SinglePartitionReadQuery.java
index e595fcb10a..8fb612f554 100644
--- a/src/java/org/apache/cassandra/db/SinglePartitionReadQuery.java
+++ b/src/java/org/apache/cassandra/db/SinglePartitionReadQuery.java
@@ -74,7 +74,7 @@ public interface SinglePartitionReadQuery extends ReadQuery
                                                   ColumnFilter columnFilter,
                                                   ClusteringIndexFilter filter)
     {
-        return create(metadata, nowInSec, columnFilter, RowFilter.NONE, DataLimits.NONE, key, filter);
+        return create(metadata, nowInSec, columnFilter, RowFilter.none(), DataLimits.NONE, key, filter);
     }
 
     /**
diff --git a/src/java/org/apache/cassandra/db/filter/RowFilter.java b/src/java/org/apache/cassandra/db/filter/RowFilter.java
index 6d717016b0..94c175314e 100644
--- a/src/java/org/apache/cassandra/db/filter/RowFilter.java
+++ b/src/java/org/apache/cassandra/db/filter/RowFilter.java
@@ -23,6 +23,8 @@ import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 import com.google.common.base.Objects;
 import org.slf4j.Logger;
@@ -63,7 +65,6 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
     private static final Logger logger = LoggerFactory.getLogger(RowFilter.class);
 
     public static final Serializer serializer = new Serializer();
-    public static final RowFilter NONE = new CQLFilter(Collections.emptyList());
 
     protected final List<Expression> expressions;
 
@@ -82,6 +83,11 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
         return new CQLFilter(new ArrayList<>(capacity));
     }
 
+    public static RowFilter none()
+    {
+        return CQLFilter.NONE;
+    }
+
     public SimpleExpression add(ColumnMetadata def, Operator op, ByteBuffer value)
     {
         SimpleExpression expression = new SimpleExpression(def, op, value);
@@ -229,7 +235,7 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
     {
         assert expressions.contains(expression);
         if (expressions.size() == 1)
-            return RowFilter.NONE;
+            return RowFilter.none();
 
         List<Expression> newExpressions = new ArrayList<>(expressions.size() - 1);
         for (Expression e : expressions)
@@ -261,6 +267,11 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
         return withNewExpressions(Collections.emptyList());
     }
 
+    public RowFilter restrict(Predicate<Expression> filter)
+    {
+        return withNewExpressions(expressions.stream().filter(filter).collect(Collectors.toList()));
+    }
+
     protected abstract RowFilter withNewExpressions(List<Expression> expressions);
 
     public boolean isEmpty()
@@ -303,6 +314,8 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
 
     private static class CQLFilter extends RowFilter
     {
+        static CQLFilter NONE = new CQLFilter(Collections.emptyList());
+
         private CQLFilter(List<Expression> expressions)
         {
             super(expressions);
diff --git a/src/java/org/apache/cassandra/db/view/TableViews.java b/src/java/org/apache/cassandra/db/view/TableViews.java
index 619e9f8566..d1c1db478e 100644
--- a/src/java/org/apache/cassandra/db/view/TableViews.java
+++ b/src/java/org/apache/cassandra/db/view/TableViews.java
@@ -462,7 +462,7 @@ public class TableViews extends AbstractCollection<View>
         // TODO: we could still make sense to special case for when there is a single view and a small number of updates (and
         // no deletions). Indeed, in that case we could check whether any of the update modify any of the restricted regular
         // column, and if that's not the case we could use view filter. We keep it simple for now though.
-        RowFilter rowFilter = RowFilter.NONE;
+        RowFilter rowFilter = RowFilter.none();
         return SinglePartitionReadCommand.create(metadata, nowInSec, queriedColumns, rowFilter, DataLimits.NONE, key, clusteringFilter);
     }
 
diff --git a/src/java/org/apache/cassandra/db/virtual/SystemViewsKeyspace.java b/src/java/org/apache/cassandra/db/virtual/SystemViewsKeyspace.java
index 12df6c7790..6a718c7145 100644
--- a/src/java/org/apache/cassandra/db/virtual/SystemViewsKeyspace.java
+++ b/src/java/org/apache/cassandra/db/virtual/SystemViewsKeyspace.java
@@ -19,6 +19,8 @@ package org.apache.cassandra.db.virtual;
 
 import com.google.common.collect.ImmutableList;
 
+import org.apache.cassandra.index.sai.virtual.StorageAttachedIndexTables;
+
 import static org.apache.cassandra.schema.SchemaConstants.VIRTUAL_VIEWS;
 
 public final class SystemViewsKeyspace extends VirtualKeyspace
@@ -51,6 +53,7 @@ public final class SystemViewsKeyspace extends VirtualKeyspace
                     .add(new LogMessagesTable(VIRTUAL_VIEWS))
                     .add(new SnapshotsTable(VIRTUAL_VIEWS))
                     .addAll(LocalRepairTables.getAll(VIRTUAL_VIEWS))
+                    .addAll(StorageAttachedIndexTables.getAll(VIRTUAL_VIEWS))
                     .build());
     }
 }
diff --git a/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java b/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java
index 57993a69a6..873ced8e39 100644
--- a/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java
+++ b/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java
@@ -203,6 +203,12 @@ public class Murmur3Partitioner implements IPartitioner
             return token;
         }
 
+        @Override
+        public long getLongValue()
+        {
+            return token;
+        }
+
         @Override
         public double size(Token next)
         {
diff --git a/src/java/org/apache/cassandra/dht/Token.java b/src/java/org/apache/cassandra/dht/Token.java
index 3543dabc0e..ff574a6af8 100644
--- a/src/java/org/apache/cassandra/dht/Token.java
+++ b/src/java/org/apache/cassandra/dht/Token.java
@@ -124,6 +124,21 @@ public abstract class Token implements RingPosition<Token>, Serializable
     abstract public long getHeapSize();
     abstract public Object getTokenValue();
 
+    /**
+     * This method exists so that callers can access the primitive {@code long} value for this {@link Token}, if
+     * one exits. It is especially useful when the auto-boxing induced by a call to {@link #getTokenValue()} would
+     * be unacceptable for reasons of performance.
+     *
+     * @return the primitive {@code long} value of this token, if one exists
+     *
+     * @throws UnsupportedOperationException if this {@link Token} is not backed by a primitive {@code long} value
+     */
+    public long getLongValue()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+
     /**
      * Produce a weakly prefix-free byte-comparable representation of the token, i.e. such a sequence of bytes that any
      * pair x, y of valid tokens of this type and any bytes b1, b2 between 0x10 and 0xEF,
diff --git a/src/java/org/apache/cassandra/exceptions/RequestFailureReason.java b/src/java/org/apache/cassandra/exceptions/RequestFailureReason.java
index ca642986cf..51415facd5 100644
--- a/src/java/org/apache/cassandra/exceptions/RequestFailureReason.java
+++ b/src/java/org/apache/cassandra/exceptions/RequestFailureReason.java
@@ -35,7 +35,8 @@ public enum RequestFailureReason
     TIMEOUT                  (2),
     INCOMPATIBLE_SCHEMA      (3),
     READ_SIZE                (4),
-    NODE_DOWN                (5);
+    NODE_DOWN                (5),
+    INDEX_NOT_AVAILABLE      (6);
 
     public static final Serializer serializer = new Serializer();
 
diff --git a/src/java/org/apache/cassandra/gms/ApplicationState.java b/src/java/org/apache/cassandra/gms/ApplicationState.java
index 3f44bcc05b..4d30ce4cc9 100644
--- a/src/java/org/apache/cassandra/gms/ApplicationState.java
+++ b/src/java/org/apache/cassandra/gms/ApplicationState.java
@@ -57,6 +57,7 @@ public enum ApplicationState
      **/
     SSTABLE_VERSIONS,
     DISK_USAGE,
+    INDEX_STATUS,
     // DO NOT EDIT OR REMOVE PADDING STATES BELOW - only add new states above.  See CASSANDRA-16484
     X1,
     X2,
diff --git a/src/java/org/apache/cassandra/gms/VersionedValue.java b/src/java/org/apache/cassandra/gms/VersionedValue.java
index 7c64278721..031df1b8a3 100644
--- a/src/java/org/apache/cassandra/gms/VersionedValue.java
+++ b/src/java/org/apache/cassandra/gms/VersionedValue.java
@@ -269,6 +269,11 @@ public class VersionedValue implements Comparable<VersionedValue>
             return new VersionedValue(VersionedValue.SHUTDOWN + VersionedValue.DELIMITER + value);
         }
 
+        public VersionedValue indexStatus(String status)
+        {
+            return new VersionedValue(status);
+        }
+
         public VersionedValue datacenter(String dcId)
         {
             return new VersionedValue(dcId);
diff --git a/src/java/org/apache/cassandra/index/Index.java b/src/java/org/apache/cassandra/index/Index.java
index c556cb9b65..152606069e 100644
--- a/src/java/org/apache/cassandra/index/Index.java
+++ b/src/java/org/apache/cassandra/index/Index.java
@@ -512,19 +512,19 @@ public interface Index
          * Notification of the start of a partition update.
          * This event always occurs before any other during the update.
          */
-        public void begin();
+        default void begin() {}
 
         /**
          * Notification of a top level partition delete.
          */
-        public void partitionDelete(DeletionTime deletionTime);
+        default void partitionDelete(DeletionTime deletionTime) {}
 
         /**
          * Notification of a RangeTombstone.
          * An update of a single partition may contain multiple RangeTombstones,
          * and a notification will be passed for each of them.
          */
-        public void rangeTombstone(RangeTombstone tombstone);
+        default void rangeTombstone(RangeTombstone tombstone) {}
 
         /**
          * Notification that a new row was inserted into the Memtable holding the partition.
@@ -534,7 +534,7 @@ public interface Index
          *
          * @param row the Row being inserted into the base table's Memtable.
          */
-        public void insertRow(Row row);
+        default void insertRow(Row row) {}
 
         /**
          * Notification of a modification to a row in the base table's Memtable.
@@ -555,7 +555,7 @@ public interface Index
          * @param newRowData data that was not present in the existing row and is being inserted
          *                   into the base table's Memtable
          */
-        public void updateRow(Row oldRowData, Row newRowData);
+        default void updateRow(Row oldRowData, Row newRowData) {}
 
         /**
          * Notification that a row was removed from the partition.
@@ -573,13 +573,13 @@ public interface Index
          *
          * @param row data being removed from the base table
          */
-        public void removeRow(Row row);
+        default void removeRow(Row row) {}
 
         /**
          * Notification of the end of the partition update.
          * This event always occurs after all others for the particular update.
          */
-        public void finish();
+        default void finish() {}
     }
 
     /*
diff --git a/src/java/org/apache/cassandra/index/IndexStatusManager.java b/src/java/org/apache/cassandra/index/IndexStatusManager.java
new file mode 100644
index 0000000000..0a872b5a0d
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/IndexStatusManager.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.cassandra.concurrent.ExecutorPlus;
+import org.apache.cassandra.db.ConsistencyLevel;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.exceptions.ReadFailureException;
+import org.apache.cassandra.exceptions.RequestFailureReason;
+import org.apache.cassandra.gms.ApplicationState;
+import org.apache.cassandra.gms.Gossiper;
+import org.apache.cassandra.gms.VersionedValue;
+import org.apache.cassandra.locator.Endpoints;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.FBUtilities;
+import org.json.simple.JSONValue;
+import org.json.simple.parser.ParseException;
+
+import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
+
+/**
+ * Handles the status of an index across the ring, updating the status per index and endpoint
+ * in a per-endpoint map.
+ *
+ * Peer status changes are recieved via the {@link StorageService} {@link org.apache.cassandra.gms.IEndpointStateChangeSubscriber}.
+ *
+ * Local status changes are propagated to the {@link Gossiper} using an async executor.
+ */
+public class IndexStatusManager
+{
+    private static final Logger logger = LoggerFactory.getLogger(IndexStatusManager.class);
+
+    public static final IndexStatusManager instance = new IndexStatusManager();
+
+    // executes index status propagation task asynchronously to avoid potential deadlock on SIM
+    private final ExecutorPlus statusPropagationExecutor = executorFactory().withJmxInternal()
+                                                                            .sequential("StatusPropagationExecutor");
+    // used to produce a status string from the current endpoint states in JSON format for Gossip propagation
+    private final ObjectMapper objectMapper = new ObjectMapper();
+
+    /**
+     * A map of per-endpoint index statuses: the key of inner map is the identifier "keyspace.index"
+     */
+    public final Map<InetAddressAndPort, Map<String, Index.Status>> peerIndexStatus = new HashMap<>();
+
+    private IndexStatusManager()
+    {}
+
+    /**
+     * Remove endpoints whose indexes are not queryable for the specified {@link Index.QueryPlan}.
+     *
+     * @param liveEndpoints current live endpoints where non-queryable endpoints will be removed
+     * @param keyspace to be queried
+     * @param indexQueryPlan index query plan used in the read command
+     * @param level consistency level of read command
+     */
+    public <E extends Endpoints<E>> E filterForQuery(E liveEndpoints, Keyspace keyspace, Index.QueryPlan indexQueryPlan, ConsistencyLevel level)
+    {
+        E queryableEndpoints = liveEndpoints.filter(replica -> {
+
+            for (Index index : indexQueryPlan.getIndexes())
+            {
+                Index.Status status = getIndexStatus(replica.endpoint(), keyspace.getName(), index.getIndexMetadata().name);
+                if (!index.isQueryable(status))
+                    return false;
+            }
+
+            return true;
+        });
+
+        int initial = liveEndpoints.size();
+        int filtered = queryableEndpoints.size();
+
+        // Throw ReadFailureException if read request cannot satisfy Consistency Level due to non-queryable indexes.
+        // It is to provide a better UX, compared to throwing UnavailableException when the nodes are actually alive.
+        if (initial != filtered)
+        {
+            int required = level.blockFor(keyspace.getReplicationStrategy());
+            if (required <= initial && required > filtered)
+            {
+                Map<InetAddressAndPort, RequestFailureReason> failureReasons = new HashMap<>();
+                liveEndpoints.without(queryableEndpoints.endpoints())
+                             .forEach(replica -> failureReasons.put(replica.endpoint(), RequestFailureReason.INDEX_NOT_AVAILABLE));
+
+                throw new ReadFailureException(level, filtered, required, false, failureReasons);
+            }
+        }
+
+        return queryableEndpoints;
+    }
+
+    /**
+     * Recieve a new index status map from a peer. This will include the status for all the indexes on the peer.
+     *
+     * @param endpoint the {@link InetAddressAndPort} the index status map is coming from
+     * @param versionedValue the {@link VersionedValue} containing the index status map
+     */
+    public synchronized void receivePeerIndexStatus(InetAddressAndPort endpoint, VersionedValue versionedValue)
+    {
+        try
+        {
+            if (versionedValue == null)
+                return;
+            if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
+                return;
+
+            @SuppressWarnings("unchecked")
+            Map<String, String> peerStatus = (Map<String, String>) JSONValue.parseWithException(versionedValue.value);
+            Map<String, Index.Status> indexStatus = new HashMap<>();
+
+            for (Map.Entry<String, String> e : peerStatus.entrySet())
+            {
+                String keyspaceIndex = e.getKey();
+                Index.Status status = Index.Status.valueOf(e.getValue());
+                indexStatus.put(keyspaceIndex, status);
+            }
+
+            Map<String, Index.Status> oldStatus = peerIndexStatus.put(endpoint, indexStatus);
+            Map<String, Index.Status> updated = updatedIndexStatuses(oldStatus, indexStatus);
+            Set<String> removed = removedIndexStatuses(oldStatus, indexStatus);
+            if (!updated.isEmpty() || !removed.isEmpty())
+                logger.debug("Received index status for peer {}:\n    Updated: {}\n    Removed: {}",
+                             endpoint, updated, removed);
+        }
+        catch (ParseException | IllegalArgumentException e)
+        {
+            logger.warn("Unable to parse index status: {}", e.getMessage());
+        }
+    }
+
+    /**
+     * Propagate a new index status to the ring. The new index status is added to the current index status map
+     * and the whole map is sent to the ring as a {@link VersionedValue}.
+     *
+     * @param keyspace the keyspace name for the index
+     * @param index the index name
+     * @param status the new {@link Index.Status}
+     */
+    public synchronized void propagateLocalIndexStatus(String keyspace, String index, Index.Status status)
+    {
+        try
+        {
+            Map<String, Index.Status> states = peerIndexStatus.computeIfAbsent(FBUtilities.getBroadcastAddressAndPort(),
+                                                                               k -> new HashMap<>());
+            String keyspaceIndex = identifier(keyspace, index);
+
+            if (status == Index.Status.DROPPED)
+                states.remove(keyspaceIndex);
+            else
+                states.put(keyspaceIndex, status);
+
+            // Don't try and propagate if the gossiper isn't enabled. This is primarily for tests where the
+            // Gossiper has not been started. If we attempt to propagate when not started an exception is
+            // logged and this causes a number of dtests to fail.
+            if (Gossiper.instance.isEnabled())
+            {
+                String newStatus = objectMapper.writeValueAsString(states);
+                statusPropagationExecutor.submit(() -> {
+                    // schedule gossiper update asynchronously to avoid potential deadlock when another thread is holding
+                    // gossiper taskLock.
+                    VersionedValue value = StorageService.instance.valueFactory.indexStatus(newStatus);
+                    Gossiper.instance.addLocalApplicationState(ApplicationState.INDEX_STATUS, value);
+                });
+            }
+        }
+        catch (Throwable e)
+        {
+            logger.warn("Unable to propagate index status: {}", e.getMessage());
+        }
+    }
+
+    @VisibleForTesting
+    public synchronized Index.Status getIndexStatus(InetAddressAndPort peer, String keyspace, String index)
+    {
+        return peerIndexStatus.getOrDefault(peer, Collections.emptyMap())
+                              .getOrDefault(identifier(keyspace, index), Index.Status.UNKNOWN);
+    }
+
+    /**
+     * Returns the names of indexes that are present in oldStatus but absent in newStatus.
+     */
+    private @Nonnull Set<String> removedIndexStatuses(@Nullable Map<String, Index.Status> oldStatus,
+                                                      @Nonnull Map<String, Index.Status> newStatus)
+    {
+        if (oldStatus == null)
+            return Collections.emptySet();
+        Set<String> result = new HashSet<>(oldStatus.keySet());
+        result.removeAll(newStatus.keySet());
+        return result;
+    }
+
+    /**
+     * Returns a new map containing only the entries from newStatus that differ from corresponding entries in oldStatus.
+     */
+    private @Nonnull Map<String, Index.Status> updatedIndexStatuses(@Nullable Map<String, Index.Status> oldStatus,
+                                                                    @Nonnull Map<String, Index.Status> newStatus)
+    {
+        Map<String, Index.Status> delta = new HashMap<>();
+        for (Map.Entry<String, Index.Status> e : newStatus.entrySet())
+        {
+            if (oldStatus == null || e.getValue() != oldStatus.get(e.getKey()))
+                delta.put(e.getKey(), e.getValue());
+        }
+        return delta;
+    }
+
+    private String identifier(String keyspace, String index)
+    {
+        return keyspace + '.' + index;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/SecondaryIndexManager.java b/src/java/org/apache/cassandra/index/SecondaryIndexManager.java
index 1aec1da481..35272266ed 100644
--- a/src/java/org/apache/cassandra/index/SecondaryIndexManager.java
+++ b/src/java/org/apache/cassandra/index/SecondaryIndexManager.java
@@ -48,7 +48,6 @@ import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.apache.cassandra.concurrent.FutureTask;
 import org.apache.cassandra.concurrent.ImmediateExecutor;
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
@@ -214,7 +213,6 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
                : blockingExecutor.submit(reloadTask);
     }
 
-    @SuppressWarnings("unchecked")
     private synchronized Future<Void> createIndex(IndexMetadata indexDef, boolean isNewCF)
     {
         final Index index = createInstance(indexDef);
@@ -654,7 +652,8 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
      * @param isNewCF {@code true} if this method is invoked when initializing a new table/columnfamily (i.e. loading a CF at startup),
      * {@code false} for all other cases (i.e. newly added index)
      */
-    private synchronized void markIndexesBuilding(Set<Index> indexes, boolean isFullRebuild, boolean isNewCF)
+    @VisibleForTesting
+    public synchronized void markIndexesBuilding(Set<Index> indexes, boolean isFullRebuild, boolean isNewCF)
     {
         String keyspaceName = baseCfs.keyspace.getName();
 
@@ -766,6 +765,8 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
         writableIndexes.remove(indexName);
         needsFullRebuild.remove(indexName);
         inProgressBuilds.remove(indexName);
+        // remove existing indexing status
+        IndexStatusManager.instance.propagateLocalIndexStatus(keyspace.getName(), indexName, Index.Status.DROPPED);
     }
 
     public Index getIndexByName(String indexName)
@@ -779,8 +780,8 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
         if (indexDef.isCustom())
         {
             assert indexDef.options != null;
-            // Find any aliases to the fully qualified index class name:
-            String className = IndexMetadata.expandAliases(indexDef.options.get(IndexTarget.CUSTOM_INDEX_OPTION_NAME));
+            // Get the fully qualified index class name from the index metadata
+            String className = indexDef.getIndexClassName();
             assert !Strings.isNullOrEmpty(className);
 
             try
@@ -943,7 +944,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
             SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(baseCfs.metadata(),
                                                                                FBUtilities.nowInSeconds(),
                                                                                ColumnFilter.selection(columns),
-                                                                               RowFilter.NONE,
+                                                                               RowFilter.none(),
                                                                                DataLimits.NONE,
                                                                                key,
                                                                                new ClusteringIndexSliceFilter(Slices.ALL, false));
@@ -1180,12 +1181,20 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
         return indexes.stream().map(i -> i.getIndexMetadata().name).collect(Collectors.joining(","));
     }
 
-
     public Optional<Index> getBestIndexFor(RowFilter.Expression expression)
     {
         return indexes.values().stream().filter((i) -> i.supportsExpression(expression.column(), expression.operator())).findFirst();
     }
 
+    public <T extends Index> Set<T> getBestIndexFor(RowFilter.Expression expression, Class<T> indexType)
+    {
+        return indexes.values()
+                      .stream()
+                      .filter(i -> indexType.isInstance(i) && i.supportsExpression(expression.column(), expression.operator()))
+                      .map(indexType::cast)
+                      .collect(Collectors.toSet());
+    }
+
     /**
      * Called at write time to ensure that values present in the update
      * are valid according to the rules of all registered indexes which
@@ -1711,9 +1720,13 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
 
     public void makeIndexNonQueryable(Index index, Index.Status status)
     {
+        if (status == Index.Status.BUILD_SUCCEEDED)
+            throw new IllegalStateException("Index cannot be marked non-queryable with status " + status);
+
         String name = index.getIndexMetadata().name;
         if (indexes.get(name) == index)
         {
+            IndexStatusManager.instance.propagateLocalIndexStatus(keyspace.getName(), name, status);
             if (!index.isQueryable(status))
                 queryableIndexes.remove(name);
         }
@@ -1721,9 +1734,13 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
 
     public void makeIndexQueryable(Index index, Index.Status status)
     {
+        if (status != Index.Status.BUILD_SUCCEEDED)
+            throw new IllegalStateException("Index cannot be marked queryable with status " + status);
+
         String name = index.getIndexMetadata().name;
         if (indexes.get(name) == index)
         {
+            IndexStatusManager.instance.propagateLocalIndexStatus(keyspace.getName(), name, status);
             if (index.isQueryable(status))
             {
                 if (queryableIndexes.add(name))
diff --git a/src/java/org/apache/cassandra/index/internal/composites/CompositesSearcher.java b/src/java/org/apache/cassandra/index/internal/composites/CompositesSearcher.java
index 457dee1306..675b84923e 100644
--- a/src/java/org/apache/cassandra/index/internal/composites/CompositesSearcher.java
+++ b/src/java/org/apache/cassandra/index/internal/composites/CompositesSearcher.java
@@ -118,7 +118,7 @@ public class CompositesSearcher extends CassandraIndexSearcher
                         dataCmd = SinglePartitionReadCommand.create(index.baseCfs.metadata(),
                                                                     command.nowInSec(),
                                                                     command.columnFilter(),
-                                                                    RowFilter.NONE,
+                                                                    RowFilter.none(),
                                                                     DataLimits.NONE,
                                                                     partitionKey,
                                                                     command.clusteringIndexFilter(partitionKey));
diff --git a/src/java/org/apache/cassandra/index/sai/IndexContext.java b/src/java/org/apache/cassandra/index/sai/IndexContext.java
new file mode 100644
index 0000000000..ca67f3d340
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/IndexContext.java
@@ -0,0 +1,453 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.marshal.UUIDType;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sai.memory.MemtableIndex;
+import org.apache.cassandra.index.sai.metrics.IndexMetrics;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyFactory;
+import org.apache.cassandra.index.sai.utils.KeyRangeIterator;
+import org.apache.cassandra.index.sai.utils.KeyRangeUnionIterator;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * Manage metadata for each column index.
+ */
+public class IndexContext
+{
+    private static final Set<AbstractType<?>> EQ_ONLY_TYPES = ImmutableSet.of(UTF8Type.instance,
+                                                                              AsciiType.instance,
+                                                                              BooleanType.instance,
+                                                                              UUIDType.instance);
+
+    private final AbstractType<?> partitionKeyType;
+    private final ClusteringComparator clusteringComparator;
+
+    private final String keyspace;
+    private final String table;
+    private final ColumnMetadata columnMetadata;
+    private final IndexTarget.Type indexType;
+    private final AbstractType<?> validator;
+
+    // Config can be null if the column context is "fake" (i.e. created for a filtering expression).
+    @Nullable
+    private final IndexMetadata config;
+
+    private final ConcurrentMap<Memtable, MemtableIndex> liveMemtableIndexMap;
+
+    private final IndexMetrics indexMetrics;
+
+    private final AbstractAnalyzer.AnalyzerFactory indexAnalyzerFactory;
+    private final AbstractAnalyzer.AnalyzerFactory queryAnalyzerFactory;
+    private final PrimaryKeyFactory primaryKeyFactory;
+
+    public IndexContext(String keyspace,
+                        String table,
+                        AbstractType<?> partitionKeyType,
+                        ClusteringComparator clusteringComparator,
+                        ColumnMetadata columnMetadata,
+                        IndexTarget.Type indexType,
+                        @Nullable IndexMetadata config)
+    {
+        this.keyspace = Objects.requireNonNull(keyspace);
+        this.table = Objects.requireNonNull(table);
+        this.partitionKeyType = Objects.requireNonNull(partitionKeyType);
+        this.clusteringComparator = Objects.requireNonNull(clusteringComparator);
+        this.columnMetadata = Objects.requireNonNull(columnMetadata);
+        this.indexType = Objects.requireNonNull(indexType);
+        this.validator = TypeUtil.cellValueType(columnMetadata, indexType);
+        this.primaryKeyFactory = PrimaryKey.factory(clusteringComparator);
+
+        this.config = config;
+        this.indexMetrics = config == null ? null : new IndexMetrics(this);
+        this.liveMemtableIndexMap = config == null ? null : new ConcurrentHashMap<>();
+
+        // We currently only support the NoOpAnalyzer
+        this.indexAnalyzerFactory = AbstractAnalyzer.fromOptions(getValidator(), Collections.emptyMap());
+        this.queryAnalyzerFactory = AbstractAnalyzer.fromOptions(getValidator(), Collections.emptyMap());
+    }
+
+    public AbstractType<?> keyValidator()
+    {
+        return partitionKeyType;
+    }
+
+    public PrimaryKeyFactory keyFactory()
+    {
+        return primaryKeyFactory;
+    }
+
+    public ClusteringComparator comparator()
+    {
+        return clusteringComparator;
+    }
+
+    public String getKeyspace()
+    {
+        return keyspace;
+    }
+
+    public String getTable()
+    {
+        return table;
+    }
+
+    public IndexMetadata getIndexMetadata()
+    {
+        return config;
+    }
+
+    public long index(DecoratedKey key, Row row, Memtable mt)
+    {
+        assert config != null : "Attempt to index on a non-indexing context";
+
+        MemtableIndex current = liveMemtableIndexMap.get(mt);
+
+        // We expect the relevant IndexMemtable to be present most of the time, so only make the
+        // call to computeIfAbsent() if it's not. (see https://bugs.openjdk.java.net/browse/JDK-8161372)
+        MemtableIndex target = (current != null)
+                               ? current
+                               : liveMemtableIndexMap.computeIfAbsent(mt, memtable -> new MemtableIndex(this));
+
+        long start = Clock.Global.nanoTime();
+
+        long bytes = 0;
+
+        if (isNonFrozenCollection())
+        {
+            Iterator<ByteBuffer> bufferIterator = getValuesOf(row, FBUtilities.nowInSeconds());
+            if (bufferIterator != null)
+            {
+                while (bufferIterator.hasNext())
+                {
+                    ByteBuffer value = bufferIterator.next();
+                    bytes += target.index(key, row.clustering(), value);
+                }
+            }
+        }
+        else
+        {
+            ByteBuffer value = getValueOf(key, row, FBUtilities.nowInSeconds());
+            target.index(key, row.clustering(), value);
+        }
+        indexMetrics.memtableIndexWriteLatency.update(Clock.Global.nanoTime() - start, TimeUnit.NANOSECONDS);
+        return bytes;
+    }
+
+    public void renewMemtable(Memtable renewed)
+    {
+        assert liveMemtableIndexMap != null : "Attempt to renew memtable on non-indexing context";
+
+        for (Memtable memtable : liveMemtableIndexMap.keySet())
+        {
+            // remove every index but the one that corresponds to the post-truncate Memtable
+            if (renewed != memtable)
+            {
+                liveMemtableIndexMap.remove(memtable);
+            }
+        }
+    }
+
+    public void discardMemtable(Memtable discarded)
+    {
+        assert liveMemtableIndexMap != null : "Attempt to discard memtable from non-indexing context";
+
+        liveMemtableIndexMap.remove(discarded);
+    }
+
+    public KeyRangeIterator searchMemtableIndexes(Expression e, AbstractBounds<PartitionPosition> keyRange)
+    {
+        assert liveMemtableIndexMap != null : "Attempt to perform search on non-indexing context";
+
+        Collection<MemtableIndex> memtableIndexes = liveMemtableIndexMap.values();
+
+        if (memtableIndexes.isEmpty())
+        {
+            return KeyRangeIterator.empty();
+        }
+
+        KeyRangeIterator.Builder builder = KeyRangeUnionIterator.builder(memtableIndexes.size());
+
+        for (MemtableIndex memtableIndex : memtableIndexes)
+        {
+            builder.add(memtableIndex.search(e, keyRange));
+        }
+
+        return builder.build();
+    }
+
+    public long liveMemtableWriteCount()
+    {
+        assert liveMemtableIndexMap != null : "Attempt to get metrics from non-indexing context";
+
+        return liveMemtableIndexMap.values().stream().mapToLong(MemtableIndex::writeCount).sum();
+    }
+
+    public long estimatedMemIndexMemoryUsed()
+    {
+        assert liveMemtableIndexMap != null : "Attempt to get metrics from non-indexing context";
+
+        return liveMemtableIndexMap.values().stream().mapToLong(MemtableIndex::estimatedMemoryUsed).sum();
+    }
+
+    public ColumnMetadata getDefinition()
+    {
+        return columnMetadata;
+    }
+
+    public AbstractType<?> getValidator()
+    {
+        return validator;
+    }
+
+    public boolean isNonFrozenCollection()
+    {
+        return TypeUtil.isNonFrozenCollection(columnMetadata.type);
+    }
+
+    public boolean isFrozen()
+    {
+        return TypeUtil.isFrozen(columnMetadata.type);
+    }
+
+    public String getColumnName()
+    {
+        return columnMetadata.name.toString();
+    }
+
+    @Nullable
+    public String getIndexName()
+    {
+        return config == null ? null : config.name;
+    }
+
+    /**
+     * Returns an {@code AnalyzerFactory} for use by the write path to transform incoming literal
+     * during indexing. The analyzers can be tokenising or non-tokenising. Tokenising analyzers
+     * will split the incoming terms into multiple terms in the index while non-tokenising analyzers
+     * will not split the incoming term but will transform the term (e.g. case-insensitive)
+     */
+    public AbstractAnalyzer.AnalyzerFactory getIndexAnalyzerFactory()
+    {
+        return indexAnalyzerFactory;
+    }
+
+    /**
+     * Return an {@code AnalyzerFactory} for use by the query path to transform query terms before
+     * searching for them in the index. This can be the same as the indexAnalyzerFactory.
+     */
+    public AbstractAnalyzer.AnalyzerFactory getQueryAnalyzerFactory()
+    {
+        return queryAnalyzerFactory;
+    }
+
+    public boolean isIndexed()
+    {
+        return config != null;
+    }
+
+    /**
+     * Called when index is dropped. Clear all live in-memory indexes and close
+     * analyzer factories.
+     */
+    public void invalidate()
+    {
+        if (liveMemtableIndexMap != null)
+            liveMemtableIndexMap.clear();
+        if (indexMetrics != null)
+            indexMetrics.release();
+        indexAnalyzerFactory.close();
+        if (queryAnalyzerFactory != indexAnalyzerFactory)
+        {
+            queryAnalyzerFactory.close();
+        }
+    }
+
+    public boolean supports(Operator op)
+    {
+        if (op == Operator.LIKE ||
+            op == Operator.LIKE_CONTAINS ||
+            op == Operator.LIKE_PREFIX ||
+            op == Operator.LIKE_MATCHES ||
+            op == Operator.LIKE_SUFFIX) return false;
+
+        Expression.IndexOperator operator = Expression.IndexOperator.valueOf(op);
+
+        if (isNonFrozenCollection())
+        {
+            if (indexType == IndexTarget.Type.KEYS) return operator == Expression.IndexOperator.CONTAINS_KEY;
+            if (indexType == IndexTarget.Type.VALUES) return operator == Expression.IndexOperator.CONTAINS_VALUE;
+            return indexType == IndexTarget.Type.KEYS_AND_VALUES && operator == Expression.IndexOperator.EQ;
+        }
+
+        if (indexType == IndexTarget.Type.FULL)
+            return operator == Expression.IndexOperator.EQ;
+
+        AbstractType<?> validator = getValidator();
+
+        if (operator != Expression.IndexOperator.EQ && EQ_ONLY_TYPES.contains(validator)) return false;
+
+        // RANGE only applicable to non-literal indexes
+        return (operator != null) && !(TypeUtil.isLiteral(validator) && operator == Expression.IndexOperator.RANGE);
+    }
+
+    public ByteBuffer getValueOf(DecoratedKey key, Row row, int nowInSecs)
+    {
+        if (row == null)
+            return null;
+
+        switch (columnMetadata.kind)
+        {
+            case PARTITION_KEY:
+                return partitionKeyType instanceof CompositeType
+                       ? CompositeType.extractComponent(key.getKey(), columnMetadata.position())
+                       : key.getKey();
+            case CLUSTERING:
+                // skip indexing of static clustering when regular column is indexed
+                return row.isStatic() ? null : row.clustering().bufferAt(columnMetadata.position());
+
+            // treat static cell retrieval the same was as regular
+            // only if row kind is STATIC otherwise return null
+            case STATIC:
+                if (!row.isStatic())
+                    return null;
+            case REGULAR:
+                Cell<?> cell = row.getCell(columnMetadata);
+                return cell == null || !cell.isLive(nowInSecs) ? null : cell.buffer();
+
+            default:
+                return null;
+        }
+    }
+
+    public Iterator<ByteBuffer> getValuesOf(Row row, int nowInSecs)
+    {
+        if (row == null)
+            return null;
+
+        switch (columnMetadata.kind)
+        {
+            // treat static cell retrieval the same was as regular
+            // only if row kind is STATIC otherwise return null
+            case STATIC:
+                if (!row.isStatic())
+                    return null;
+            case REGULAR:
+                return TypeUtil.collectionIterator(validator,
+                                                   row.getComplexColumnData(columnMetadata),
+                                                   columnMetadata,
+                                                   indexType,
+                                                   nowInSecs);
+
+            default:
+                return null;
+        }
+    }
+
+    @Override
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this)
+                          .add("columnName", getColumnName())
+                          .add("indexName", getIndexName())
+                          .toString();
+    }
+
+    public boolean isLiteral()
+    {
+        return TypeUtil.isLiteral(getValidator());
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (obj == this)
+            return true;
+
+        if (!(obj instanceof IndexContext))
+            return false;
+
+        IndexContext other = (IndexContext) obj;
+
+        return Objects.equals(columnMetadata, other.columnMetadata) &&
+               (indexType == other.indexType) &&
+               Objects.equals(config, other.config) &&
+               Objects.equals(partitionKeyType, other.partitionKeyType) &&
+               Objects.equals(clusteringComparator, other.clusteringComparator);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(columnMetadata, indexType, config, partitionKeyType, clusteringComparator);
+    }
+
+    /**
+     * A helper method for constructing consistent log messages for specific column indexes.
+     *
+     * Example: For the index "idx" in keyspace "ks" on table "tb", calling this method with the raw message
+     * "Flushing new index segment..." will produce...
+     *
+     * "[ks.tb.idx] Flushing new index segment..."
+     *
+     * @param message The raw content of a logging message, without information identifying it with an index.
+     *
+     * @return A log message with the proper keyspace, table and index name prepended to it.
+     */
+    public String logMessage(String message)
+    {
+        // Index names are unique only within a keyspace.
+        return String.format("[%s.%s.%s] %s", keyspace, table, config == null ? "?" : config.name, message);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/QueryContext.java b/src/java/org/apache/cassandra/index/sai/QueryContext.java
new file mode 100644
index 0000000000..4fff4160d3
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/QueryContext.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai;
+
+import java.util.concurrent.TimeUnit;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.exceptions.QueryCancelledException;
+import org.apache.cassandra.utils.Clock;
+
+/**
+ * Tracks state relevant to the execution of a single query, including metrics and timeout monitoring.
+ *
+ * Fields here are non-volatile, as they are accessed from a single thread.
+ */
+@NotThreadSafe
+public class QueryContext
+{
+    private static final boolean DISABLE_TIMEOUT = Boolean.getBoolean("cassandra.sai.test.disable.timeout");
+
+    private final ReadCommand readCommand;
+    private final long queryStartTimeNanos;
+
+    public final long executionQuotaNano;
+
+    public long partitionsRead = 0;
+    public long rowsFiltered = 0;
+
+    public long queryTimeouts = 0;
+
+    public QueryContext(ReadCommand readCommand, long executionQuotaMs)
+    {
+        this.readCommand = readCommand;
+        executionQuotaNano = TimeUnit.MILLISECONDS.toNanos(executionQuotaMs);
+        queryStartTimeNanos = Clock.Global.nanoTime();
+    }
+
+    public long totalQueryTimeNs()
+    {
+        return Clock.Global.nanoTime() - queryStartTimeNanos;
+    }
+
+    public void checkpoint()
+    {
+        if (totalQueryTimeNs() >= executionQuotaNano && !DISABLE_TIMEOUT)
+        {
+            queryTimeouts++;
+            throw new QueryCancelledException(readCommand);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/README.md b/src/java/org/apache/cassandra/index/sai/README.md
new file mode 100644
index 0000000000..45cdfeedab
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/README.md
@@ -0,0 +1,115 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Storage-Attached Indexing
+
+## Overview
+Storage-attached indexing is a column based secondary indexing apparatus for Cassandra.
+
+The project was inspired by SASI (SSTable-Attached Secondary Indexes) and retains some of its high-level
+architectural character (and even some actual code), but makes significant improvements in a number of areas:
+
+- The on-disk/SSTable index formats for both string and numeric data have been completely replaced. Strings are indexed
+  on disk using our proprietary on-disk byte-ordered trie data structure, while numeric types are indexed using Lucene's
+  balanced kd-tree.
+- While indexes continue to be managed at the column level from the user's perspective, the storage design at the column
+  index level is row-based, with related offset and token information stored only once at the SSTable level. This
+  drastically reduces our on-disk footprint when several columns are indexed on the same table.
+- Tracing, metrics, virtual table-based metadata and snapshot-based backup/restore are supported out of the box.
+
+Many similarities with standard secondary indexes remain:
+
+- The full set of C* consistency levels is supported for both reads and writes.
+- Index updates are synchronous with mutations and do not require any kind of read-before-write.
+- Queries are implemented on the back of C* range reads.
+- Paging is supported.
+- Only token ordering of results is supported.
+- Index builds are visible to operators as compactions and are executed on compaction threads.
+- All DML and DDL statements are CQL-based.
+- Single-node management operations are available via nodetool. (ex. stop & rebuild_index)
+
+## Quick Start
+
+The following short tutorial will get you up-and-running with storage-attached indexing.
+
+### Build and Start Cassandra
+
+Follow the instructions to build and start Cassandra in README.asc in root folder of the Cassandra repository
+
+### Create a Simple Data Model
+
+1.) Run the following DDL statements to create a table and two indexes:
+
+`CREATE KEYSPACE test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'};`
+
+`USE test;`
+
+`CREATE TABLE person (id int, name text, age int, PRIMARY KEY (id));`
+
+`CREATE CUSTOM INDEX ON person (name) USING 'StorageAttachedIndex' WITH OPTIONS = {'case_sensitive': false};`
+
+`CREATE CUSTOM INDEX ON person (age) USING 'StorageAttachedIndex';`
+
+2.) Add some data.
+
+`INSERT INTO person (id, name, age) VALUES (1, 'John', 21);`
+
+`INSERT INTO person (id, name, age) VALUES (2, 'john', 50);`
+
+`INSERT INTO person (id, name, age) VALUES (3, 'Boris', 43);`
+
+`INSERT INTO person (id, name, age) VALUES (4, 'Caleb', 34);`
+
+### Make Some Queries
+
+1.) Query for everyone named "John", ignoring case.
+
+`SELECT * FROM person WHERE name = 'John';`
+
+```
+ id | age | name
+----+-----+------
+  1 |  21 | John
+  2 |  50 | john
+```
+
+2.) Query for everyone between the ages of 18 and 25.
+
+`SELECT * FROM person WHERE age >= 18 AND age <= 35;`
+
+```
+ id | age | name
+----+-----+-------
+  1 |  21 |  John
+  4 |  34 | Caleb
+```
+
+## Contributors
+
+- Marc Selwan
+- Caleb Rackliffe
+- Zhao Yang
+- Jason Rutherglen
+- Maciej Zasada
+- Andres de la Peña
+- Mike Adamson
+- Zahir Patni
+- Tomek Lasica
+- Berenguer Blasi
+- Rocco Varela
+- Piotr Kołaczkowski
diff --git a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java
new file mode 100644
index 0000000000..6c79e1f08b
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java
@@ -0,0 +1,449 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture; // checkstyle: permit this import
+import java.util.concurrent.Future;
+
+import com.google.common.collect.ImmutableSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.CassandraWriteContext;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.WriteContext;
+import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.dht.ByteOrderedPartitioner;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.dht.OrderPreservingPartitioner;
+import org.apache.cassandra.dht.RandomPartitioner;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.IndexRegistry;
+import org.apache.cassandra.index.SecondaryIndexBuilder;
+import org.apache.cassandra.index.TargetParser;
+import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.index.transactions.IndexTransaction;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+
+public class StorageAttachedIndex implements Index
+{
+    private static final Logger logger = LoggerFactory.getLogger(StorageAttachedIndex.class);
+
+    private static class StorageAttachedIndexBuildingSupport implements IndexBuildingSupport
+    {
+        @Override
+        public SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs,
+                                                       Set<Index> indexes,
+                                                       Collection<SSTableReader> sstablesToRebuild,
+                                                       boolean isFullRebuild)
+        {
+            // Return an empty builder for in-memory implementation
+            return new SecondaryIndexBuilder()
+            {
+                @Override
+                public void build()
+                {
+                }
+
+                @Override
+                public CompactionInfo getCompactionInfo()
+                {
+                    return null;
+                }
+            };
+        }
+    }
+
+    // Used to build indexes on newly added SSTables:
+    private static final StorageAttachedIndexBuildingSupport INDEX_BUILDER_SUPPORT = new StorageAttachedIndexBuildingSupport();
+
+    private static final Set<String> VALID_OPTIONS = ImmutableSet.of(IndexTarget.TARGET_OPTION_NAME,
+                                                                     IndexTarget.CUSTOM_INDEX_OPTION_NAME);
+
+    public static final Set<CQL3Type> SUPPORTED_TYPES = ImmutableSet.of(CQL3Type.Native.ASCII, CQL3Type.Native.BIGINT, CQL3Type.Native.DATE,
+                                                                        CQL3Type.Native.DOUBLE, CQL3Type.Native.FLOAT, CQL3Type.Native.INT,
+                                                                        CQL3Type.Native.SMALLINT, CQL3Type.Native.TEXT, CQL3Type.Native.TIME,
+                                                                        CQL3Type.Native.TIMESTAMP, CQL3Type.Native.TIMEUUID, CQL3Type.Native.TINYINT,
+                                                                        CQL3Type.Native.UUID, CQL3Type.Native.VARCHAR, CQL3Type.Native.INET,
+                                                                        CQL3Type.Native.VARINT, CQL3Type.Native.DECIMAL, CQL3Type.Native.BOOLEAN);
+
+    private static final Set<Class<? extends IPartitioner>> ILLEGAL_PARTITIONERS =
+            ImmutableSet.of(OrderPreservingPartitioner.class, LocalPartitioner.class, ByteOrderedPartitioner.class, RandomPartitioner.class);
+
+    private final ColumnFamilyStore baseCfs;
+    private final IndexContext indexContext;
+
+    public StorageAttachedIndex(ColumnFamilyStore baseCfs, IndexMetadata config)
+    {
+        this.baseCfs = baseCfs;
+        TableMetadata tableMetadata = baseCfs.metadata();
+        Pair<ColumnMetadata, IndexTarget.Type> target = TargetParser.parse(tableMetadata, config);
+        this.indexContext = new IndexContext(tableMetadata.keyspace,
+                                             tableMetadata.name,
+                                             tableMetadata.partitionKeyType,
+                                             tableMetadata.comparator,
+                                             target.left,
+                                             target.right,
+                                             config);
+    }
+
+    /**
+     * Used via reflection in {@link IndexMetadata}
+     */
+    @SuppressWarnings({ "unused" })
+    public static Map<String, String> validateOptions(Map<String, String> options, TableMetadata metadata)
+    {
+        Map<String, String> unknown = new HashMap<>(2);
+
+        for (Map.Entry<String, String> option : options.entrySet())
+        {
+            if (!VALID_OPTIONS.contains(option.getKey()))
+            {
+                unknown.put(option.getKey(), option.getValue());
+            }
+        }
+
+        if (!unknown.isEmpty())
+        {
+            return unknown;
+        }
+
+        if (ILLEGAL_PARTITIONERS.contains(metadata.partitioner.getClass()))
+        {
+            throw new InvalidRequestException("Storage-attached index does not support the following IPartitioner implementations: " + ILLEGAL_PARTITIONERS);
+        }
+
+        String targetColumn = options.get(IndexTarget.TARGET_OPTION_NAME);
+
+        if (targetColumn == null)
+        {
+            throw new InvalidRequestException("Missing target column");
+        }
+
+        if (targetColumn.split(",").length > 1)
+        {
+            throw new InvalidRequestException("A storage-attached index cannot be created over multiple columns: " + targetColumn);
+        }
+
+        Pair<ColumnMetadata, IndexTarget.Type> target = TargetParser.parse(metadata, targetColumn);
+
+        if (target == null)
+        {
+            throw new InvalidRequestException("Failed to retrieve target column for: " + targetColumn);
+        }
+
+        // In order to support different index targets on non-frozen map, ie. KEYS, VALUE, ENTRIES, we need to put index
+        // name as part of index file name instead of column name. We only need to check that the target is different
+        // between indexes. This will only allow indexes in the same column with a different IndexTarget.Type.
+        //
+        // Note that: "metadata.indexes" already includes current index
+        if (metadata.indexes.stream().filter(index -> index.getIndexClassName().equals(StorageAttachedIndex.class.getName()))
+                            .map(index -> TargetParser.parse(metadata, index.options.get(IndexTarget.TARGET_OPTION_NAME)))
+                            .filter(Objects::nonNull).filter(t -> t.equals(target)).count() > 1)
+        {
+            throw new InvalidRequestException("Cannot create more than one storage-attached index on the same column: " + target.left);
+        }
+
+        AbstractType<?> type = TypeUtil.cellValueType(target.left, target.right);
+
+        // If we are indexing map entries we need to validate the subtypes
+        if (TypeUtil.isComposite(type))
+        {
+            for (AbstractType<?> subType : type.subTypes())
+            {
+                if (!SUPPORTED_TYPES.contains(subType.asCQL3Type()) && !TypeUtil.isFrozen(subType))
+                    throw new InvalidRequestException("Unsupported type: " + subType.asCQL3Type());
+            }
+        }
+        else if (!SUPPORTED_TYPES.contains(type.asCQL3Type()) && !TypeUtil.isFrozen(type))
+        {
+            throw new InvalidRequestException("Unsupported type: " + type.asCQL3Type());
+        }
+
+        AbstractAnalyzer.fromOptions(type, options);
+
+        return Collections.emptyMap();
+    }
+
+    @Override
+    public void register(IndexRegistry registry)
+    {
+        // index will be available for writes
+        registry.registerIndex(this, StorageAttachedIndexGroup.class, () -> new StorageAttachedIndexGroup(baseCfs));
+    }
+
+    @Override
+    public IndexMetadata getIndexMetadata()
+    {
+        return indexContext.getIndexMetadata();
+    }
+
+    @Override
+    public Callable<?> getInitializationTask()
+    {
+        // New storage-attached indexes will be available for queries after on disk index data are built.
+        // Memtable data will be indexed via flushing triggered by schema change
+        return () -> startInitialBuild(baseCfs).get();
+    }
+
+    private Future<?> startInitialBuild(ColumnFamilyStore baseCfs)
+    {
+        if (baseCfs.indexManager.isIndexQueryable(this))
+        {
+            logger.debug(indexContext.logMessage("Skipping validation and building in initialization task, as pre-join has already made the storage attached index queryable..."));
+        }
+        baseCfs.indexManager.makeIndexQueryable(this, Status.BUILD_SUCCEEDED);
+        return CompletableFuture.completedFuture(null);
+    }
+
+    @Override
+    public Callable<?> getMetadataReloadTask(IndexMetadata indexMetadata)
+    {
+        return null;
+    }
+
+    @Override
+    public Callable<?> getBlockingFlushTask()
+    {
+        return null; // storage-attached indexes are flushed alongside memtable
+    }
+
+    @Override
+    public Callable<?> getInvalidateTask()
+    {
+        return () ->
+        {
+            indexContext.invalidate();
+            return null;
+        };
+    }
+
+    @Override
+    public Callable<?> getPreJoinTask(boolean hadBootstrap)
+    {
+        /*
+         * During bootstrap, streamed SSTable are already built for existing indexes via {@link StorageAttachedIndexBuildingSupport}
+         * from {@link org.apache.cassandra.streaming.StreamReceiveTask.OnCompletionRunnable}.
+         *
+         * For indexes created during bootstrapping, we don't have to block bootstrap for them.
+         */
+
+        return this::startPreJoinTask;
+    }
+
+    private Future<?> startPreJoinTask()
+    {
+        try
+        {
+            if (baseCfs.indexManager.isIndexQueryable(this))
+                logger.debug(indexContext.logMessage("Skipping validation in pre-join task, as the initialization task has already made the index queryable..."));
+
+            baseCfs.indexManager.makeIndexQueryable(this, Status.BUILD_SUCCEEDED);
+        }
+        catch (Throwable t)
+        {
+            logger.error(indexContext.logMessage("Failed in pre-join task!"), t);
+        }
+
+        return null;
+    }
+
+    @Override
+    public Callable<?> getTruncateTask(long truncatedAt)
+    {
+        /*
+         * index files will be removed as part of base sstable lifecycle in
+         * {@link LogTransaction#delete(java.io.File)} asynchronously.
+         */
+        return null;
+    }
+
+    @Override
+    public boolean shouldBuildBlocking()
+    {
+        return true;
+    }
+
+    @Override
+    public Optional<ColumnFamilyStore> getBackingTable()
+    {
+        return Optional.empty();
+    }
+
+    @Override
+    public boolean dependsOn(ColumnMetadata column)
+    {
+        return indexContext.getDefinition().compareTo(column) == 0;
+    }
+
+    @Override
+    public boolean supportsExpression(ColumnMetadata column, Operator operator)
+    {
+        return dependsOn(column) && indexContext.supports(operator);
+    }
+
+    @Override
+    public AbstractType<?> customExpressionValueType()
+    {
+        return null;
+    }
+
+    @Override
+    public RowFilter getPostIndexQueryFilter(RowFilter filter)
+    {
+        // it should be executed from the SAI query plan, this is only used by the singleton index query plan
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public long getEstimatedResultRows()
+    {
+        throw new UnsupportedOperationException("Use StorageAttachedIndexQueryPlan#getEstimatedResultRows() instead.");
+    }
+
+    @Override
+    public boolean isQueryable(Status status)
+    {
+        // consider unknown status as queryable, because gossip may not be up-to-date for newly joining nodes.
+        return status == Status.BUILD_SUCCEEDED || status == Status.UNKNOWN;
+    }
+
+    @Override
+    public void validate(PartitionUpdate update) throws InvalidRequestException
+    {}
+
+    @Override
+    public Searcher searcherFor(ReadCommand command) throws InvalidRequestException
+    {
+        // searchers should be created from the query plan, this is only used by the singleton index query plan
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker)
+    {
+        // flush observers should be created from the index group, this is only used by the singleton index group
+        throw new UnsupportedOperationException("Storage-attached index flush observers should never be created directly.");
+    }
+
+    @Override
+    public Indexer indexerFor(DecoratedKey key,
+                              RegularAndStaticColumns columns,
+                              int nowInSec,
+                              WriteContext writeContext,
+                              IndexTransaction.Type transactionType,
+                              Memtable memtable)
+    {
+        if (transactionType == IndexTransaction.Type.UPDATE)
+        {
+            return new UpdateIndexer(key, memtable, writeContext);
+        }
+
+        // we are only interested in the data from Memtable
+        // everything else is going to be handled by SSTableWriter observers
+        return null;
+    }
+
+    @Override
+    public IndexBuildingSupport getBuildTaskSupport()
+    {
+        return INDEX_BUILDER_SUPPORT;
+    }
+
+    public IndexContext getIndexContext()
+    {
+        return indexContext;
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("%s.%s.%s", baseCfs.keyspace.getName(), baseCfs.name, getIndexMetadata() == null ? "?" : getIndexMetadata());
+    }
+
+    /**
+     * Removes this index from the {@code SecondaryIndexManager}'s set of queryable indexes.
+     */
+    public void makeIndexNonQueryable()
+    {
+        baseCfs.indexManager.makeIndexNonQueryable(this, Status.BUILD_FAILED);
+        logger.warn(indexContext.logMessage("Storage-attached index is no longer queryable. Please restart this node to repair it."));
+    }
+
+    private class UpdateIndexer implements Index.Indexer
+    {
+        private final DecoratedKey key;
+        private final Memtable memtable;
+        private final WriteContext writeContext;
+
+        UpdateIndexer(DecoratedKey key, Memtable memtable, WriteContext writeContext)
+        {
+            this.key = key;
+            this.memtable = memtable;
+            this.writeContext = writeContext;
+        }
+
+        @Override
+        public void insertRow(Row row)
+        {
+            adjustMemtableSize(indexContext.index(key, row, memtable), CassandraWriteContext.fromContext(writeContext).getGroup());
+        }
+
+        @Override
+        public void updateRow(Row oldRow, Row newRow)
+        {
+            insertRow(newRow);
+        }
+
+        void adjustMemtableSize(long additionalSpace, OpOrder.Group opGroup)
+        {
+            memtable.markExtraOnHeapUsed(additionalSpace, opGroup);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexGroup.java b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexGroup.java
new file mode 100644
index 0000000000..8871f5dc19
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexGroup.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.ThreadSafe;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Sets;
+import com.google.common.primitives.Ints;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.WriteContext;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.lifecycle.Tracker;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.sai.metrics.TableQueryMetrics;
+import org.apache.cassandra.index.sai.metrics.TableStateMetrics;
+import org.apache.cassandra.index.sai.plan.StorageAttachedIndexQueryPlan;
+import org.apache.cassandra.index.transactions.IndexTransaction;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.notifications.INotification;
+import org.apache.cassandra.notifications.INotificationConsumer;
+import org.apache.cassandra.notifications.MemtableDiscardedNotification;
+import org.apache.cassandra.notifications.MemtableRenewedNotification;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * Orchestrates building of storage-attached indices, and manages lifecycle of resources shared between them.
+ */
+@ThreadSafe
+public class StorageAttachedIndexGroup implements Index.Group, INotificationConsumer
+{
+    private final TableQueryMetrics queryMetrics;
+    private final TableStateMetrics stateMetrics;
+    private final Set<Index> indexes = Sets.newConcurrentHashSet();
+    private final ColumnFamilyStore baseCfs;
+
+    StorageAttachedIndexGroup(ColumnFamilyStore baseCfs)
+    {
+        this.baseCfs = baseCfs;
+        this.queryMetrics = new TableQueryMetrics(baseCfs.metadata());
+        this.stateMetrics = new TableStateMetrics(baseCfs.metadata(), this);
+
+        Tracker tracker = baseCfs.getTracker();
+        tracker.subscribe(this);
+    }
+
+    @Nullable
+    public static StorageAttachedIndexGroup getIndexGroup(ColumnFamilyStore cfs)
+    {
+        return (StorageAttachedIndexGroup) cfs.indexManager.getIndexGroup(StorageAttachedIndexGroup.class);
+    }
+
+    @Override
+    public Set<Index> getIndexes()
+    {
+        return indexes;
+    }
+
+    @Override
+    public void addIndex(Index index)
+    {
+        assert index instanceof StorageAttachedIndex;
+        indexes.add(index);
+    }
+
+    @Override
+    public void removeIndex(Index index)
+    {
+        assert index instanceof StorageAttachedIndex;
+        boolean removed = indexes.remove(index);
+        assert removed : "Cannot remove non-existing index " + index;
+        /*
+         * For the in-memory implementation we only need to unsubscribe from the tracker
+         */
+        if (indexes.isEmpty())
+        {
+            baseCfs.getTracker().unsubscribe(this);
+        }
+    }
+
+    @Override
+    public void invalidate()
+    {
+        // in case of dropping table, sstable contexts should already been removed by SSTableListChangedNotification.
+        queryMetrics.release();
+        stateMetrics.release();
+        baseCfs.getTracker().unsubscribe(this);
+    }
+
+    @Override
+    public boolean containsIndex(Index index)
+    {
+        return indexes.contains(index);
+    }
+
+    @Override
+    public Index.Indexer indexerFor(Predicate<Index> indexSelector,
+                                    DecoratedKey key,
+                                    RegularAndStaticColumns columns,
+                                    int nowInSec,
+                                    WriteContext ctx,
+                                    IndexTransaction.Type transactionType,
+                                    Memtable memtable)
+    {
+        final Set<Index.Indexer> indexers =
+                indexes.stream().filter(indexSelector)
+                       .map(i -> i.indexerFor(key, columns, nowInSec, ctx, transactionType, memtable))
+                       .filter(Objects::nonNull)
+                       .collect(Collectors.toSet());
+
+        return indexers.isEmpty() ? null : new Index.Indexer()
+        {
+            @Override
+            public void insertRow(Row row)
+            {
+                for (Index.Indexer indexer : indexers)
+                    indexer.insertRow(row);
+            }
+
+            @Override
+            public void updateRow(Row oldRow, Row newRow)
+            {
+                for (Index.Indexer indexer : indexers)
+                    indexer.updateRow(oldRow, newRow);
+            }
+        };
+    }
+
+    @Override
+    public StorageAttachedIndexQueryPlan queryPlanFor(RowFilter rowFilter)
+    {
+        return StorageAttachedIndexQueryPlan.create(baseCfs, queryMetrics, indexes, rowFilter);
+    }
+
+    @Override
+    public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata)
+    {
+        return null;
+    }
+
+    @Override
+    public boolean handles(IndexTransaction.Type type)
+    {
+        // to skip CleanupGCTransaction and IndexGCTransaction
+        return type == IndexTransaction.Type.UPDATE;
+    }
+
+    @Override
+    public Set<Component> getComponents()
+    {
+        return getComponents(indexes);
+    }
+
+    // TODO: Currently returns an empty set for in-memory only implementation
+    static Set<Component> getComponents(Collection<Index> indices)
+    {
+        return Collections.emptySet();
+    }
+
+    @Override
+    public void handleNotification(INotification notification, Object sender)
+    {
+        if (notification instanceof MemtableRenewedNotification)
+        {
+            indexes.forEach(index -> ((StorageAttachedIndex)index).getIndexContext().renewMemtable(((MemtableRenewedNotification) notification).renewed));
+        }
+        else if (notification instanceof MemtableDiscardedNotification)
+        {
+            indexes.forEach(index -> ((StorageAttachedIndex)index).getIndexContext().discardMemtable(((MemtableDiscardedNotification) notification).memtable));
+        }
+    }
+
+    /**
+     * @return count of queryable indexes
+     */
+    public int totalQueryableIndexCount()
+    {
+        return Ints.checkedCast(indexes.stream().filter(baseCfs.indexManager::isIndexQueryable).count());
+    }
+
+    /**
+     * @return count of indexes
+     */
+    public int totalIndexCount()
+    {
+        return indexes.size();
+    }
+
+    public TableMetadata metadata()
+    {
+        return baseCfs.metadata();
+    }
+
+    public ColumnFamilyStore table()
+    {
+        return baseCfs;
+    }
+
+    /**
+     * Simulate the index going through a restart of node
+     */
+    @VisibleForTesting
+    public void reset()
+    {
+        indexes.forEach(i -> ((StorageAttachedIndex) i).makeIndexNonQueryable());
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java b/src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java
new file mode 100644
index 0000000000..64a2f2fed5
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer;
+
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+
+public abstract class AbstractAnalyzer implements Iterator<ByteBuffer>
+{
+    protected ByteBuffer next = null;
+    String nextLiteral = null;
+
+    /**
+     * @return true if index value is transformed, e.g. normalized or lower-cased or tokenized.
+     */
+    public abstract boolean transformValue();
+
+    /**
+     * Call when tokenization is finished.  Used by the LuceneAnalyzer.
+     */
+    public void end()
+    {
+    }
+
+    /**
+     * Note: This method does not advance, as we rely on {@link #hasNext()} to buffer the next value.
+     *
+     * @return the raw value currently buffered by this iterator
+     */
+    @Override
+    public ByteBuffer next()
+    {
+        if (next == null)
+            throw new NoSuchElementException();
+        return next;
+    }
+
+    @Override
+    public void remove()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    protected abstract void resetInternal(ByteBuffer input);
+
+    public void reset(ByteBuffer input)
+    {
+        this.next = null;
+        this.nextLiteral = null;
+
+        resetInternal(input);
+    }
+
+    public interface AnalyzerFactory
+    {
+        AbstractAnalyzer create();
+
+        default void close()
+        {
+        }
+    }
+
+    public static AnalyzerFactory fromOptions(AbstractType<?> type, Map<String, String> options)
+    {
+        return NoOpAnalyzer::new;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/NoOpAnalyzer.java b/src/java/org/apache/cassandra/index/sai/analyzer/NoOpAnalyzer.java
new file mode 100644
index 0000000000..fbf76bcc25
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/NoOpAnalyzer.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.analyzer;
+
+import java.nio.ByteBuffer;
+
+import com.google.common.base.MoreObjects;
+
+/**
+ * Default NoOp tokenizer. The iterator will iterate only once returning the unmodified input
+ */
+public class NoOpAnalyzer extends AbstractAnalyzer
+{
+    private ByteBuffer input;
+    private boolean hasNext = false;
+
+    public NoOpAnalyzer() {}
+
+    @Override
+    public boolean hasNext()
+    {
+        if (hasNext)
+        {
+            this.next = input;
+            this.hasNext = false;
+            return true;
+        }
+        this.next = null;
+        return false;
+    }
+
+    @Override
+    protected void resetInternal(ByteBuffer input)
+    {
+        this.input = input;
+        this.hasNext = true;
+    }
+
+    @Override
+    public boolean transformValue()
+    {
+        return false;
+    }
+
+    @Override
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this).toString();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/memory/FilteringInMemoryKeyRangeIterator.java b/src/java/org/apache/cassandra/index/sai/memory/FilteringInMemoryKeyRangeIterator.java
new file mode 100644
index 0000000000..d54823546a
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/memory/FilteringInMemoryKeyRangeIterator.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.memory;
+
+import java.util.SortedSet;
+
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+
+/**
+ * An {@link InMemoryKeyRangeIterator} that filters the returned {@link PrimaryKey}s based on the provided keyRange
+ */
+public class FilteringInMemoryKeyRangeIterator extends InMemoryKeyRangeIterator
+{
+    private final AbstractBounds<PartitionPosition> keyRange;
+
+    public FilteringInMemoryKeyRangeIterator(SortedSet<PrimaryKey> keys, AbstractBounds<PartitionPosition> keyRange)
+    {
+        super(keys);
+        this.keyRange = keyRange;
+    }
+
+    @Override
+    protected PrimaryKey computeNext()
+    {
+        PrimaryKey key = computeNextKey();
+        while (key != null && !keyRange.contains(key.partitionKey()))
+            key = computeNextKey();
+        return key == null ? endOfData() : key;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/memory/InMemoryKeyRangeIterator.java b/src/java/org/apache/cassandra/index/sai/memory/InMemoryKeyRangeIterator.java
new file mode 100644
index 0000000000..63a08f4a19
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/memory/InMemoryKeyRangeIterator.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.memory;
+
+import java.util.PriorityQueue;
+import java.util.SortedSet;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.index.sai.utils.KeyRangeIterator;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+
+@NotThreadSafe
+public class InMemoryKeyRangeIterator extends KeyRangeIterator
+{
+    private final PriorityQueue<PrimaryKey> keys;
+    private final boolean uniqueKeys;
+    private PrimaryKey lastKey;
+
+    /**
+     * An in-memory {@link KeyRangeIterator} that uses a {@link PriorityQueue} built from a {@link SortedSet}
+     * which has no duplication as its backing store.
+     */
+    public InMemoryKeyRangeIterator(SortedSet<PrimaryKey> keys)
+    {
+        super(keys.first(), keys.last(), keys.size());
+        this.keys = new PriorityQueue<>(keys);
+        this.uniqueKeys = true;
+    }
+
+    /**
+     * An in-memory {@link KeyRangeIterator} that uses a {@link PriorityQueue} which may
+     * contain duplicated keys as its backing store.
+     */
+    public InMemoryKeyRangeIterator(PrimaryKey min, PrimaryKey max, PriorityQueue<PrimaryKey> keys)
+    {
+        super(min, max, keys.size());
+        this.keys = keys;
+        this.uniqueKeys = false;
+    }
+
+    @Override
+    protected PrimaryKey computeNext()
+    {
+        PrimaryKey key = computeNextKey();
+        return key == null ? endOfData() : key;
+    }
+
+    protected PrimaryKey computeNextKey()
+    {
+        PrimaryKey next = null;
+
+        while (!keys.isEmpty())
+        {
+            PrimaryKey key = keys.poll();
+            if (uniqueKeys)
+                return key;
+
+            if (lastKey == null || lastKey.compareTo(key) != 0)
+            {
+                next = key;
+                lastKey = key;
+                break;
+            }
+        }
+
+        return next;
+    }
+
+    @Override
+    protected void performSkipTo(PrimaryKey nextKey)
+    {
+        while (!keys.isEmpty())
+        {
+            PrimaryKey key = keys.peek();
+            if (key.compareTo(nextKey) >= 0)
+                break;
+
+            // consume smaller key
+            keys.poll();
+        }
+    }
+
+    @Override
+    public void close()
+    {}
+}
diff --git a/src/java/org/apache/cassandra/index/sai/memory/MemtableIndex.java b/src/java/org/apache/cassandra/index/sai/memory/MemtableIndex.java
new file mode 100644
index 0000000000..b7e51e675d
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/memory/MemtableIndex.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.memory;
+
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.concurrent.atomic.LongAdder;
+
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.utils.PrimaryKeys;
+import org.apache.cassandra.index.sai.utils.KeyRangeIterator;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+public class MemtableIndex
+{
+    private final TrieMemoryIndex index;
+    private final LongAdder writeCount = new LongAdder();
+    private final LongAdder estimatedMemoryUsed = new LongAdder();
+
+    public MemtableIndex(IndexContext indexContext)
+    {
+        this.index = new TrieMemoryIndex(indexContext);
+    }
+
+    public long writeCount()
+    {
+        return writeCount.sum();
+    }
+
+    public long estimatedMemoryUsed()
+    {
+        return estimatedMemoryUsed.sum();
+    }
+
+    public boolean isEmpty()
+    {
+        return getMinTerm() == null;
+    }
+
+    public ByteBuffer getMinTerm()
+    {
+        return index.getMinTerm();
+    }
+
+    public ByteBuffer getMaxTerm()
+    {
+        return index.getMaxTerm();
+    }
+
+    public long index(DecoratedKey key, Clustering<?> clustering, ByteBuffer value)
+    {
+        if (value == null || value.remaining() == 0)
+            return 0;
+
+        long ram = index.add(key, clustering, value);
+        writeCount.increment();
+        estimatedMemoryUsed.add(ram);
+        return ram;
+    }
+
+    public KeyRangeIterator search(Expression expression, AbstractBounds<PartitionPosition> keyRange)
+    {
+        return index.search(expression, keyRange);
+    }
+
+    public Iterator<Pair<ByteComparable, PrimaryKeys>> iterator()
+    {
+        return index.iterator();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java b/src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java
new file mode 100644
index 0000000000..2d48ac868f
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/memory/TrieMemoryIndex.java
@@ -0,0 +1,383 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.memory;
+
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.PriorityQueue;
+import java.util.SortedSet;
+import java.util.concurrent.atomic.LongAdder;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.util.concurrent.FastThreadLocal;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.memtable.TrieMemtable;
+import org.apache.cassandra.db.tries.InMemoryTrie;
+import org.apache.cassandra.db.tries.Trie;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeys;
+import org.apache.cassandra.index.sai.utils.KeyRangeIterator;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+
+/**
+ * This is an in-memory index using the {@link InMemoryTrie} to store a {@link ByteComparable}
+ * representation of the indexed values. Data is stored on-heap or off-heap and follows the
+ * settings of the {@link TrieMemtable} to determine where.
+ *
+ *
+ */
+public class TrieMemoryIndex
+{
+    private static final Logger logger = LoggerFactory.getLogger(TrieMemoryIndex.class);
+    private static final int MAX_RECURSIVE_KEY_LENGTH = 128;
+
+    private final IndexContext indexContext;
+    private final InMemoryTrie<PrimaryKeys> data;
+    private final PrimaryKeysReducer primaryKeysReducer;
+    private final AbstractAnalyzer.AnalyzerFactory analyzerFactory;
+    private final AbstractType<?> validator;
+    private final boolean isLiteral;
+
+    private ByteBuffer minTerm;
+    private ByteBuffer maxTerm;
+
+    public TrieMemoryIndex(IndexContext indexContext)
+    {
+        this.indexContext = indexContext;
+        this.data = new InMemoryTrie<>(TrieMemtable.BUFFER_TYPE);
+        this.primaryKeysReducer = new PrimaryKeysReducer();
+        // The use of the analyzer is within a synchronized block so can be considered thread-safe
+        this.analyzerFactory = indexContext.getIndexAnalyzerFactory();
+        this.validator = indexContext.getValidator();
+        this.isLiteral = TypeUtil.isLiteral(validator);
+    }
+
+    /**
+     * Adds an index value to the in-memory index
+     *
+     * @param key partition key for the indexed value
+     * @param clustering clustering for the indexed value
+     * @param value indexed value
+     * @return amount of heap allocated by the new value
+     */
+    public synchronized long add(DecoratedKey key, Clustering<?> clustering, ByteBuffer value)
+    {
+        AbstractAnalyzer analyzer = analyzerFactory.create();
+        try
+        {
+            value = TypeUtil.asIndexBytes(value, validator);
+            analyzer.reset(value);
+            final PrimaryKey primaryKey = indexContext.keyFactory().create(key, clustering);
+            final long initialSizeOnHeap = data.sizeOnHeap();
+            final long initialSizeOffHeap = data.sizeOffHeap();
+            final long reducerHeapSize = primaryKeysReducer.heapAllocations();
+
+            while (analyzer.hasNext())
+            {
+                final ByteBuffer term = analyzer.next();
+
+                setMinMaxTerm(term.duplicate());
+
+                final ByteComparable comparableBytes = asComparableBytes(term);
+
+                try
+                {
+                    if (term.limit() <= MAX_RECURSIVE_KEY_LENGTH)
+                    {
+                        data.putRecursive(comparableBytes, primaryKey, primaryKeysReducer);
+                    }
+                    else
+                    {
+                        data.apply(Trie.singleton(comparableBytes, primaryKey), primaryKeysReducer);
+                    }
+                }
+                catch (InMemoryTrie.SpaceExhaustedException e)
+                {
+                    throw new RuntimeException(e);
+                }
+            }
+
+            long onHeap = data.sizeOnHeap();
+            long offHeap = data.sizeOffHeap();
+            long heapAllocations = primaryKeysReducer.heapAllocations();
+            return (onHeap - initialSizeOnHeap) + (offHeap - initialSizeOffHeap) + (heapAllocations - reducerHeapSize);
+        }
+        finally
+        {
+            analyzer.end();
+        }
+    }
+
+    /**
+     * Search for an expression in the in-memory index within the {@link AbstractBounds} defined
+     * by keyRange. This can either be an exact match or a range match.
+
+     * @param expression the {@link Expression} to search for
+     * @param keyRange the {@link AbstractBounds} containing the key range to restrict the search to
+     * @return a {@link KeyRangeIterator} containing the search results
+     */
+    public KeyRangeIterator search(Expression expression, AbstractBounds<PartitionPosition> keyRange)
+    {
+        if (logger.isTraceEnabled())
+            logger.trace("Searching memtable index on expression '{}'...", expression);
+
+        switch (expression.getOp())
+        {
+            case EQ:
+            case CONTAINS_KEY:
+            case CONTAINS_VALUE:
+                return exactMatch(expression, keyRange);
+            case RANGE:
+                return rangeMatch(expression, keyRange);
+            default:
+                throw new IllegalArgumentException("Unsupported expression: " + expression);
+        }
+    }
+
+    /**
+     * Returns an {@link Iterator} over the entire dataset contained in the trie. This is used
+     * when the index is flushed to disk.
+     *
+     * @return the iterator containing the trie data
+     */
+    public Iterator<Pair<ByteComparable, PrimaryKeys>> iterator()
+    {
+        Iterator<Map.Entry<ByteComparable, PrimaryKeys>> iterator = data.entrySet().iterator();
+        return new Iterator<Pair<ByteComparable, PrimaryKeys>>()
+        {
+            @Override
+            public boolean hasNext()
+            {
+                return iterator.hasNext();
+            }
+
+            @Override
+            public Pair<ByteComparable, PrimaryKeys> next()
+            {
+                Map.Entry<ByteComparable, PrimaryKeys> entry = iterator.next();
+                return Pair.create(decode(entry.getKey()), entry.getValue());
+            }
+        };
+    }
+
+    public ByteBuffer getMinTerm()
+    {
+        return minTerm;
+    }
+
+    public ByteBuffer getMaxTerm()
+    {
+        return maxTerm;
+    }
+
+    private void setMinMaxTerm(ByteBuffer term)
+    {
+        assert term != null;
+
+        minTerm = TypeUtil.min(term, minTerm, indexContext.getValidator());
+        maxTerm = TypeUtil.max(term, maxTerm, indexContext.getValidator());
+    }
+
+    private ByteComparable asComparableBytes(ByteBuffer input)
+    {
+        return isLiteral ? version -> terminated(ByteSource.of(input, version))
+                         : version -> TypeUtil.asComparableBytes(input, validator, version);
+    }
+
+    private ByteComparable decode(ByteComparable term)
+    {
+        return isLiteral ? version -> ByteSourceInverse.unescape(ByteSource.peekable(term.asComparableBytes(version)))
+                         : term;
+
+    }
+
+    private ByteSource terminated(ByteSource src)
+    {
+        return new ByteSource()
+        {
+            boolean done = false;
+
+            @Override
+            public int next()
+            {
+                if (done)
+                    return END_OF_STREAM;
+                int n = src.next();
+                if (n != END_OF_STREAM)
+                    return n;
+
+                done = true;
+                return ByteSource.TERMINATOR;
+            }
+        };
+    }
+
+    private KeyRangeIterator exactMatch(Expression expression, AbstractBounds<PartitionPosition> keyRange)
+    {
+        ByteComparable comparableMatch = expression.lower == null ? ByteComparable.EMPTY
+                                                                  : asComparableBytes(expression.lower.value.encoded);
+        PrimaryKeys primaryKeys = data.get(comparableMatch);
+        return primaryKeys == null ? KeyRangeIterator.empty()
+                                   : new FilteringInMemoryKeyRangeIterator(primaryKeys.keys(), keyRange);
+    }
+
+    private static class Collector
+    {
+        private static final int MINIMUM_QUEUE_SIZE = 128;
+
+        // Maintain the last queue size used on this index to use for the next range match.
+        // This allows for receiving a stream of wide range queries where the queue size
+        // is larger than we would want to default the size to.
+        // TODO Investigate using a decaying histogram here to avoid the effect of outliers.
+        private static final FastThreadLocal<Integer> lastQueueSize = new FastThreadLocal<Integer>()
+        {
+            protected Integer initialValue()
+            {
+                return MINIMUM_QUEUE_SIZE;
+            }
+        };
+
+        PrimaryKey minimumKey = null;
+        PrimaryKey maximumKey = null;
+        final PriorityQueue<PrimaryKey> mergedKeys = new PriorityQueue<>(lastQueueSize.get());
+
+        final AbstractBounds<PartitionPosition> keyRange;
+
+        public Collector(AbstractBounds<PartitionPosition> keyRange)
+        {
+            this.keyRange = keyRange;
+        }
+
+        public void processContent(PrimaryKeys keys)
+        {
+            if (keys.isEmpty())
+                return;
+
+            SortedSet<PrimaryKey> primaryKeys = keys.keys();
+
+            // shortcut to avoid generating iterator
+            if (primaryKeys.size() == 1)
+            {
+                processKey(primaryKeys.first());
+                return;
+            }
+
+            // skip entire partition keys if they don't overlap
+            if (!keyRange.right.isMinimum() && primaryKeys.first().partitionKey().compareTo(keyRange.right) > 0
+                || primaryKeys.last().partitionKey().compareTo(keyRange.left) < 0)
+                return;
+
+            primaryKeys.forEach(this::processKey);
+        }
+
+        public void updateLastQueueSize()
+        {
+            lastQueueSize.set(Math.max(MINIMUM_QUEUE_SIZE, mergedKeys.size()));
+        }
+
+        private void processKey(PrimaryKey key)
+        {
+            if (keyRange.contains(key.partitionKey()))
+            {
+                mergedKeys.add(key);
+
+                minimumKey = minimumKey == null ? key : key.compareTo(minimumKey) < 0 ? key : minimumKey;
+                maximumKey = maximumKey == null ? key : key.compareTo(maximumKey) > 0 ? key : maximumKey;
+            }
+        }
+    }
+
+    private KeyRangeIterator rangeMatch(Expression expression, AbstractBounds<PartitionPosition> keyRange)
+    {
+        ByteComparable lowerBound, upperBound;
+        boolean lowerInclusive, upperInclusive;
+        if (expression.lower != null)
+        {
+            lowerBound = asComparableBytes(expression.lower.value.encoded);
+            lowerInclusive = expression.lower.inclusive;
+        }
+        else
+        {
+            lowerBound = ByteComparable.EMPTY;
+            lowerInclusive = false;
+        }
+
+        if (expression.upper != null)
+        {
+            upperBound = asComparableBytes(expression.upper.value.encoded);
+            upperInclusive = expression.upper.inclusive;
+        }
+        else
+        {
+            upperBound = null;
+            upperInclusive = false;
+        }
+
+        Collector cd = new Collector(keyRange);
+
+        data.subtrie(lowerBound, lowerInclusive, upperBound, upperInclusive)
+            .values()
+            .forEach(cd::processContent);
+
+        if (cd.mergedKeys.isEmpty())
+        {
+            return KeyRangeIterator.empty();
+        }
+
+        cd.updateLastQueueSize();
+
+        return new InMemoryKeyRangeIterator(cd.minimumKey, cd.maximumKey, cd.mergedKeys);
+    }
+
+    private static class PrimaryKeysReducer implements InMemoryTrie.UpsertTransformer<PrimaryKeys, PrimaryKey>
+    {
+        private final LongAdder heapAllocations = new LongAdder();
+
+        @Override
+        public PrimaryKeys apply(PrimaryKeys existing, PrimaryKey neww)
+        {
+            if (existing == null)
+            {
+                existing = new PrimaryKeys();
+                heapAllocations.add(existing.unsharedHeapSize());
+            }
+            heapAllocations.add(existing.add(neww));
+            return existing;
+        }
+
+        long heapAllocations()
+        {
+            return heapAllocations.longValue();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/metrics/AbstractMetrics.java b/src/java/org/apache/cassandra/index/sai/metrics/AbstractMetrics.java
new file mode 100644
index 0000000000..d9fbb8cc72
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/metrics/AbstractMetrics.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.metrics;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.cassandra.metrics.CassandraMetricsRegistry;
+import org.apache.cassandra.metrics.DefaultNameFactory;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+public abstract class AbstractMetrics
+{
+    public static final String TYPE = "StorageAttachedIndex";
+
+    protected final String keyspace;
+    protected final String table;
+    private final String index;
+    private final String scope;
+    protected final List<CassandraMetricsRegistry.MetricName> tracked = new ArrayList<>();
+
+    AbstractMetrics(String keyspace, String table, String scope)
+    {
+        this(keyspace, table, null, scope);
+    }
+
+    AbstractMetrics(String keyspace, String table, String index, String scope)
+    {
+        assert keyspace != null && table != null : "SAI metrics must include keyspace and table";
+        this.keyspace = keyspace;
+        this.table = table;
+        this.index = index;
+        this.scope = scope;
+    }
+
+    public void release()
+    {
+        tracked.forEach(Metrics::remove);
+        tracked.clear();
+    }
+
+    protected CassandraMetricsRegistry.MetricName createMetricName(String name)
+    {
+        return createMetricName(name, scope);
+    }
+
+    protected CassandraMetricsRegistry.MetricName createMetricName(String name, String scope)
+    {
+        String metricScope = keyspace + '.' + table;
+        if (index != null)
+        {
+            metricScope += '.' + index;
+        }
+        metricScope += '.' + scope + '.' + name;
+
+        CassandraMetricsRegistry.MetricName metricName = new CassandraMetricsRegistry.MetricName(DefaultNameFactory.GROUP_NAME,
+                                                                                                 TYPE, name, metricScope, createMBeanName(name, scope));
+        tracked.add(metricName);
+        return metricName;
+    }
+
+    private String createMBeanName(String name, String scope)
+    {
+        StringBuilder builder = new StringBuilder();
+        builder.append(DefaultNameFactory.GROUP_NAME);
+        builder.append(":type=").append(TYPE);
+        builder.append(',').append("keyspace=").append(keyspace);
+        builder.append(',').append("table=").append(table);
+        if (index != null)
+            builder.append(',').append("index=").append(index);
+        builder.append(',').append("scope=").append(scope);
+        builder.append(',').append("name=").append(name);
+        return builder.toString();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/metrics/IndexMetrics.java b/src/java/org/apache/cassandra/index/sai/metrics/IndexMetrics.java
new file mode 100644
index 0000000000..d8d485d6b1
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/metrics/IndexMetrics.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.metrics;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Timer;
+import org.apache.cassandra.index.sai.IndexContext;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+public class IndexMetrics extends AbstractMetrics
+{
+    public final Timer memtableIndexWriteLatency;
+    public final Gauge<Long> liveMemtableIndexWriteCount;
+    public final Gauge<Long> memtableIndexBytes;
+
+    public IndexMetrics(IndexContext context)
+    {
+        super(context.getKeyspace(), context.getTable(), context.getIndexName(), "IndexMetrics");
+
+        memtableIndexWriteLatency = Metrics.timer(createMetricName("MemtableIndexWriteLatency"));
+        liveMemtableIndexWriteCount = Metrics.register(createMetricName("LiveMemtableIndexWriteCount"), context::liveMemtableWriteCount);
+        memtableIndexBytes = Metrics.register(createMetricName("MemtableIndexBytes"), context::estimatedMemIndexMemoryUsed);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/metrics/TableQueryMetrics.java b/src/java/org/apache/cassandra/index/sai/metrics/TableQueryMetrics.java
new file mode 100644
index 0000000000..5cf3a94a36
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/metrics/TableQueryMetrics.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.metrics;
+
+import java.util.concurrent.TimeUnit;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Timer;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.tracing.Tracing;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+public class TableQueryMetrics extends AbstractMetrics
+{
+    public static final String TABLE_QUERY_METRIC_TYPE = "TableQueryMetrics";
+
+    private final PerQueryMetrics perQueryMetrics;
+    private final Counter totalQueryTimeouts;
+    private final Counter totalPartitionReads;
+    private final Counter totalRowsFiltered;
+    private final Counter totalQueriesCompleted;
+
+    public TableQueryMetrics(TableMetadata table)
+    {
+        super(table.keyspace, table.name, TABLE_QUERY_METRIC_TYPE);
+
+        perQueryMetrics = new PerQueryMetrics(table);
+
+        totalPartitionReads = Metrics.counter(createMetricName("TotalPartitionReads"));
+        totalRowsFiltered = Metrics.counter(createMetricName("TotalRowsFiltered"));
+        totalQueriesCompleted = Metrics.counter(createMetricName("TotalQueriesCompleted"));
+        totalQueryTimeouts = Metrics.counter(createMetricName("TotalQueryTimeouts"));
+    }
+
+    public void record(QueryContext queryContext)
+    {
+        if (queryContext.queryTimeouts > 0)
+        {
+            assert queryContext.queryTimeouts == 1;
+
+            totalQueryTimeouts.inc();
+        }
+
+        perQueryMetrics.record(queryContext);
+    }
+
+    public void release()
+    {
+        super.release();
+        perQueryMetrics.release();
+    }
+
+    public class PerQueryMetrics extends AbstractMetrics
+    {
+        private final Timer queryLatency;
+
+        /**
+         * Global metrics for all indexes hit during the query.
+         */
+        private final Histogram partitionReads;
+        private final Histogram rowsFiltered;
+
+        public PerQueryMetrics(TableMetadata table)
+        {
+            super(table.keyspace, table.name, "PerQuery");
+
+            queryLatency = Metrics.timer(createMetricName("QueryLatency"));
+
+            partitionReads = Metrics.histogram(createMetricName("PartitionReads"), false);
+            rowsFiltered = Metrics.histogram(createMetricName("RowsFiltered"), false);
+        }
+
+        public void record(QueryContext queryContext)
+        {
+            final long totalQueryTimeNs = queryContext.totalQueryTimeNs();
+            queryLatency.update(totalQueryTimeNs, TimeUnit.NANOSECONDS);
+            final long queryLatencyMicros = TimeUnit.NANOSECONDS.toMicros(totalQueryTimeNs);
+
+            final long partitionsRead = queryContext.partitionsRead;
+            final long rowsFiltered = queryContext.rowsFiltered;
+
+            partitionReads.update(partitionsRead);
+            totalPartitionReads.inc(partitionsRead);
+
+            this.rowsFiltered.update(rowsFiltered);
+            totalRowsFiltered.inc(rowsFiltered);
+
+            if (Tracing.isTracing())
+            {
+                Tracing.trace("Index query accessed memtable indexes and took {} microseconds.",
+                              queryLatencyMicros);
+            }
+
+            totalQueriesCompleted.inc();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/metrics/TableStateMetrics.java b/src/java/org/apache/cassandra/index/sai/metrics/TableStateMetrics.java
new file mode 100644
index 0000000000..d9d00a9e3f
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/metrics/TableStateMetrics.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.metrics;
+
+import com.codahale.metrics.Gauge;
+import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
+import org.apache.cassandra.schema.TableMetadata;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+public class TableStateMetrics extends AbstractMetrics
+{
+    public static final String TABLE_STATE_METRIC_TYPE = "TableStateMetrics";
+
+    public final Gauge<Integer> totalIndexCount;
+    public final Gauge<Integer> totalQueryableIndexCount;
+
+    public TableStateMetrics(TableMetadata table, StorageAttachedIndexGroup group)
+    {
+        super(table.keyspace, table.name, TABLE_STATE_METRIC_TYPE);
+        totalQueryableIndexCount = Metrics.register(createMetricName("TotalQueryableIndexCount"), group::totalQueryableIndexCount);
+        totalIndexCount = Metrics.register(createMetricName("TotalIndexCount"), group::totalIndexCount);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/plan/Expression.java b/src/java/org/apache/cassandra/index/sai/plan/Expression.java
new file mode 100644
index 0000000000..c2b2fa8830
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/plan/Expression.java
@@ -0,0 +1,378 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.plan;
+
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+
+public class Expression
+{
+    private static final Logger logger = LoggerFactory.getLogger(Expression.class);
+
+    public enum IndexOperator
+    {
+        EQ, RANGE, CONTAINS_KEY, CONTAINS_VALUE;
+
+        public static IndexOperator valueOf(Operator operator)
+        {
+            switch (operator)
+            {
+                case EQ:
+                    return EQ;
+
+                case CONTAINS:
+                    return CONTAINS_VALUE; // non-frozen map: value contains term;
+
+                case CONTAINS_KEY:
+                    return CONTAINS_KEY; // non-frozen map: value contains key term;
+
+                case LT:
+                case GT:
+                case LTE:
+                case GTE:
+                    return RANGE;
+
+                default:
+                    return null;
+            }
+        }
+    }
+
+    public final AbstractAnalyzer.AnalyzerFactory analyzerFactory;
+
+    public final IndexContext context;
+    public final AbstractType<?> validator;
+
+    @VisibleForTesting
+    protected IndexOperator operator;
+
+    public Bound lower, upper;
+    // The upperInclusive and lowerInclusive flags are maintained separately to the inclusive flags
+    // in the upper and lower bounds because the upper and lower bounds have their inclusivity relaxed
+    // if the datatype being filtered is rounded in the index. These flags are used in the post-filtering
+    // process to remove values equal to the bounds.
+    public boolean upperInclusive, lowerInclusive;
+
+    public Expression(IndexContext indexContext)
+    {
+        this.context = indexContext;
+        this.analyzerFactory = indexContext.getQueryAnalyzerFactory();
+        this.validator = indexContext.getValidator();
+    }
+
+    /**
+     * This adds an operation to the current {@link Expression} instance and
+     * returns the current instance.
+     *
+     * @param op the CQL3 operation
+     * @param value the expression value
+     * @return the current expression with the added operation
+     */
+    public Expression add(Operator op, ByteBuffer value)
+    {
+        boolean lowerInclusive, upperInclusive;
+        // If the type supports rounding then we need to make sure that index
+        // range search is always inclusive, otherwise we run the risk of
+        // missing values that are within the exclusive range but are rejected
+        // because their rounded value is the same as the value being queried.
+        lowerInclusive = upperInclusive = TypeUtil.supportsRounding(validator);
+        switch (op)
+        {
+            case EQ:
+            case CONTAINS:
+            case CONTAINS_KEY:
+                lower = new Bound(value, validator, true);
+                upper = lower;
+                operator = IndexOperator.valueOf(op);
+                break;
+
+            case LTE:
+                if (context.getDefinition().isReversedType())
+                {
+                    this.lowerInclusive = true;
+                    lowerInclusive = true;
+                }
+                else
+                {
+                    this.upperInclusive = true;
+                    upperInclusive = true;
+                }
+            case LT:
+                operator = IndexOperator.RANGE;
+                if (context.getDefinition().isReversedType())
+                    lower = new Bound(value, validator, lowerInclusive);
+                else
+                    upper = new Bound(value, validator, upperInclusive);
+                break;
+
+            case GTE:
+                if (context.getDefinition().isReversedType())
+                {
+                    this.upperInclusive = true;
+                    upperInclusive = true;
+                }
+                else
+                {
+                    this.lowerInclusive = true;
+                    lowerInclusive = true;
+                }
+            case GT:
+                operator = IndexOperator.RANGE;
+                if (context.getDefinition().isReversedType())
+                    upper = new Bound(value, validator, upperInclusive);
+                else
+                    lower = new Bound(value, validator, lowerInclusive);
+                break;
+        }
+
+        assert operator != null;
+
+        return this;
+    }
+
+    /**
+     * Used in post-filtering to determine is an indexed value matches the expression
+     */
+    public boolean isSatisfiedBy(ByteBuffer columnValue)
+    {
+        if (!TypeUtil.isValid(columnValue, validator))
+        {
+            logger.error(context.logMessage("Value is not valid for indexed column {} with {}"), context.getColumnName(), validator);
+            return false;
+        }
+
+        Value value = new Value(columnValue, validator);
+
+        if (lower != null)
+        {
+            // suffix check
+            if (TypeUtil.isLiteral(validator))
+                return validateStringValue(value.raw, lower.value.raw);
+            else
+            {
+                // range or (not-)equals - (mainly) for numeric values
+                int cmp = TypeUtil.comparePostFilter(lower.value, value, validator);
+
+                // in case of EQ lower == upper
+                if (operator == IndexOperator.EQ || operator == IndexOperator.CONTAINS_KEY || operator == IndexOperator.CONTAINS_VALUE)
+                    return cmp == 0;
+
+                if (cmp > 0 || (cmp == 0 && !lowerInclusive))
+                    return false;
+            }
+        }
+
+        if (upper != null && lower != upper)
+        {
+            // string (prefix or suffix) check
+            if (TypeUtil.isLiteral(validator))
+                return validateStringValue(value.raw, upper.value.raw);
+            else
+            {
+                // range - mainly for numeric values
+                int cmp = TypeUtil.comparePostFilter(upper.value, value, validator);
+                return (cmp > 0 || (cmp == 0 && upperInclusive));
+            }
+        }
+
+        return true;
+    }
+
+    private boolean validateStringValue(ByteBuffer columnValue, ByteBuffer requestedValue)
+    {
+        AbstractAnalyzer analyzer = analyzerFactory.create();
+        analyzer.reset(columnValue.duplicate());
+        try
+        {
+            while (analyzer.hasNext())
+            {
+                final ByteBuffer term = analyzer.next();
+
+                boolean isMatch = false;
+                switch (operator)
+                {
+                    case EQ:
+                    case CONTAINS_KEY:
+                    case CONTAINS_VALUE:
+                        isMatch = validator.compare(term, requestedValue) == 0;
+                        break;
+                    case RANGE:
+                        isMatch = isLowerSatisfiedBy(term) && isUpperSatisfiedBy(term);
+                        break;
+                }
+
+                if (isMatch)
+                    return true;
+            }
+            return false;
+        }
+        finally
+        {
+            analyzer.end();
+        }
+    }
+
+    public IndexOperator getOp()
+    {
+        return operator;
+    }
+
+    private boolean hasLower()
+    {
+        return lower != null;
+    }
+
+    private boolean hasUpper()
+    {
+        return upper != null;
+    }
+
+    private boolean isLowerSatisfiedBy(ByteBuffer value)
+    {
+        if (!hasLower())
+            return true;
+
+        int cmp = validator.compare(value, lower.value.raw);
+        return cmp > 0 || cmp == 0 && lower.inclusive;
+    }
+
+    private boolean isUpperSatisfiedBy(ByteBuffer value)
+    {
+        if (!hasUpper())
+            return true;
+
+        int cmp = validator.compare(value, upper.value.raw);
+        return cmp < 0 || cmp == 0 && upper.inclusive;
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("Expression{name: %s, op: %s, lower: (%s, %s), upper: (%s, %s)}",
+                             context.getColumnName(),
+                             operator,
+                             lower == null ? "null" : validator.getString(lower.value.raw),
+                             lower != null && lower.inclusive,
+                             upper == null ? "null" : validator.getString(upper.value.raw),
+                             upper != null && upper.inclusive);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return new HashCodeBuilder().append(context.getColumnName())
+                                    .append(operator)
+                                    .append(validator)
+                                    .append(lower).append(upper).build();
+    }
+
+    @Override
+    public boolean equals(Object other)
+    {
+        if (!(other instanceof Expression))
+            return false;
+
+        if (this == other)
+            return true;
+
+        Expression o = (Expression) other;
+
+        return Objects.equals(context.getColumnName(), o.context.getColumnName())
+               && validator.equals(o.validator)
+               && operator == o.operator
+               && Objects.equals(lower, o.lower)
+               && Objects.equals(upper, o.upper);
+    }
+
+    /**
+     * A representation of a column value in its raw and encoded form.
+     */
+    public static class Value
+    {
+        public final ByteBuffer raw;
+        public final ByteBuffer encoded;
+
+        public Value(ByteBuffer value, AbstractType<?> type)
+        {
+            this.raw = value;
+            this.encoded = TypeUtil.asIndexBytes(value, type);
+        }
+
+        @Override
+        public boolean equals(Object other)
+        {
+            if (!(other instanceof Value))
+                return false;
+
+            Value o = (Value) other;
+            return raw.equals(o.raw) && encoded.equals(o.encoded);
+        }
+
+        @Override
+        public int hashCode()
+        {
+            HashCodeBuilder builder = new HashCodeBuilder();
+            builder.append(raw);
+            builder.append(encoded);
+            return builder.toHashCode();
+        }
+    }
+
+    public static class Bound
+    {
+        public final Value value;
+        public final boolean inclusive;
+
+        public Bound(ByteBuffer value, AbstractType<?> type, boolean inclusive)
+        {
+            this.value = new Value(value, type);
+            this.inclusive = inclusive;
+        }
+
+        @Override
+        public boolean equals(Object other)
+        {
+            if (!(other instanceof Bound))
+                return false;
+
+            Bound o = (Bound) other;
+            return value.equals(o.value) && inclusive == o.inclusive;
+        }
+
+        @Override
+        public int hashCode()
+        {
+            HashCodeBuilder builder = new HashCodeBuilder();
+            builder.append(value);
+            builder.append(inclusive);
+            return builder.toHashCode();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/plan/FilterTree.java b/src/java/org/apache/cassandra/index/sai/plan/FilterTree.java
new file mode 100644
index 0000000000..33990ad043
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/plan/FilterTree.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.plan;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+
+import com.google.common.collect.ListMultimap;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.ColumnMetadata.Kind;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.index.sai.plan.Operation.BooleanOperator;
+
+/**
+ * Tree-like structure to filter base table data using indexed expressions and non-user-defined filters.
+ *
+ * This is needed because:
+ * 1. SAI doesn't index tombstones, base data may have been shadowed.
+ * 2. Replica filter protecting may fetch data that doesn't match index expressions.
+ */
+public class FilterTree
+{
+    protected final BooleanOperator op;
+    protected final ListMultimap<ColumnMetadata, Expression> expressions;
+    protected final List<FilterTree> children = new ArrayList<>();
+
+    FilterTree(BooleanOperator operation,
+               ListMultimap<ColumnMetadata, Expression> expressions)
+    {
+        this.op = operation;
+        this.expressions = expressions;
+    }
+
+    void addChild(FilterTree child)
+    {
+        children.add(child);
+    }
+
+    public boolean isSatisfiedBy(DecoratedKey key, Unfiltered unfiltered, Row staticRow)
+    {
+        boolean result = localSatisfiedBy(key, unfiltered, staticRow);
+
+        for (FilterTree child : children)
+            result = op.apply(result, child.isSatisfiedBy(key, unfiltered, staticRow));
+
+        return result;
+    }
+
+    private boolean localSatisfiedBy(DecoratedKey key, Unfiltered unfiltered, Row staticRow)
+    {
+        if (unfiltered == null || !unfiltered.isRow())
+            return false;
+
+        final int now = FBUtilities.nowInSeconds();
+        boolean result = op == BooleanOperator.AND;
+
+        Iterator<ColumnMetadata> columnIterator = expressions.keySet().iterator();
+        while (columnIterator.hasNext())
+        {
+            ColumnMetadata column = columnIterator.next();
+            Row row = column.kind == Kind.STATIC ? staticRow : (Row) unfiltered;
+
+            // If there is a column with multiple expressions that can mean an OR or (in the case of map
+            // collections) it can mean different map indexes.
+            List<Expression> filters = expressions.get(column);
+
+            // We do a reverse iteration over the filters because NOT_EQ operations will be at the end
+            // of the filter list, and we want to check them first.
+            ListIterator<Expression> filterIterator = filters.listIterator(filters.size());
+            while (filterIterator.hasPrevious())
+            {
+                Expression filter = filterIterator.previous();
+
+                if (TypeUtil.isNonFrozenCollection(column.type))
+                {
+                    Iterator<ByteBuffer> valueIterator = filter.context.getValuesOf(row, now);
+                    result = op.apply(result, collectionMatch(valueIterator, filter));
+                }
+                else
+                {
+                    ByteBuffer value = filter.context.getValueOf(key, row, now);
+                    result = op.apply(result, singletonMatch(value, filter));
+                }
+
+                // If the operation is an AND then exit early if we get a single false
+                if (op == BooleanOperator.AND && !result)
+                    return false;
+            }
+        }
+        return result;
+    }
+
+    private boolean singletonMatch(ByteBuffer value, Expression filter)
+    {
+        return value != null && filter.isSatisfiedBy(value);
+    }
+
+    private boolean collectionMatch(Iterator<ByteBuffer> valueIterator, Expression filter)
+    {
+        if (valueIterator == null)
+            return false;
+
+        while (valueIterator.hasNext())
+        {
+            ByteBuffer value = valueIterator.next();
+            if (value == null)
+                continue;
+
+            if (filter.isSatisfiedBy(value))
+                return true;
+        }
+        return false;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/plan/Operation.java b/src/java/org/apache/cassandra/index/sai/plan/Operation.java
new file mode 100644
index 0000000000..cefc4268c1
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/plan/Operation.java
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.plan;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.BiFunction;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.ListMultimap;
+
+import org.apache.cassandra.cql3.Operator;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sai.utils.KeyRangeIterator;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.schema.ColumnMetadata;
+
+public class Operation
+{
+    public enum BooleanOperator
+    {
+        AND((a, b) -> a & b);
+
+        private final BiFunction<Boolean, Boolean, Boolean> func;
+
+        BooleanOperator(BiFunction<Boolean, Boolean, Boolean> func)
+        {
+            this.func = func;
+        }
+
+        public boolean apply(boolean a, boolean b)
+        {
+            return func.apply(a, b);
+        }
+    }
+
+    @VisibleForTesting
+    protected static ListMultimap<ColumnMetadata, Expression> buildIndexExpressions(QueryController controller,
+                                                                                    BooleanOperator booleanOperator,
+                                                                                    List<RowFilter.Expression> expressions)
+    {
+        ListMultimap<ColumnMetadata, Expression> analyzed = ArrayListMultimap.create();
+
+        // sort all the expressions in the operation by name and priority of the logical operator
+        // this gives us an efficient way to handle inequality and combining into ranges without extra processing
+        // and converting expressions from one type to another.
+        expressions.sort((a, b) -> {
+            int cmp = a.column().compareTo(b.column());
+            return cmp == 0 ? -Integer.compare(getPriority(a.operator()), getPriority(b.operator())) : cmp;
+        });
+
+        for (final RowFilter.Expression e : expressions)
+        {
+            IndexContext indexContext = controller.getContext(e);
+            List<Expression> perColumn = analyzed.get(e.column());
+
+            AbstractAnalyzer analyzer = indexContext.getQueryAnalyzerFactory().create();
+            try
+            {
+                analyzer.reset(e.getIndexValue().duplicate());
+
+                // EQ can have multiple expressions e.g. text = "Hello World",
+                // becomes text = "Hello" OR text = "World" because "space" is always interpreted as a split point (by analyzer),
+                // CONTAINS/CONTAINS_KEY are always treated as multiple expressions since they currently only targetting
+                // collections.
+                boolean isMultiExpression = false;
+                switch (e.operator())
+                {
+                    case EQ:
+                        // EQ operator will always be a multiple expression because it is being used by
+                        // map entries
+                        isMultiExpression = indexContext.isNonFrozenCollection();
+                        break;
+
+                    case CONTAINS:
+                    case CONTAINS_KEY:
+                        isMultiExpression = true;
+                        break;
+                }
+                if (isMultiExpression)
+                {
+                    while (analyzer.hasNext())
+                    {
+                        final ByteBuffer token = analyzer.next();
+                        perColumn.add(new Expression(indexContext).add(e.operator(), token.duplicate()));
+                    }
+                }
+                else
+                // "range" or not-equals operator, combines both bounds together into the single expression,
+                // if operation of the group is AND, otherwise we are forced to create separate expressions,
+                // not-equals is combined with the range iff operator is AND.
+                {
+                    Expression range;
+                    if (perColumn.size() == 0 || booleanOperator != BooleanOperator.AND)
+                    {
+                        range = new Expression(indexContext);
+                        perColumn.add(range);
+                    }
+                    else
+                    {
+                        range = Iterables.getLast(perColumn);
+                    }
+
+                    if (!TypeUtil.isLiteral(indexContext.getValidator()))
+                    {
+                        range.add(e.operator(), e.getIndexValue().duplicate());
+                    }
+                    else
+                    {
+                        while (analyzer.hasNext())
+                        {
+                            ByteBuffer term = analyzer.next();
+                            range.add(e.operator(), term.duplicate());
+                        }
+                    }
+                }
+            }
+            finally
+            {
+                analyzer.end();
+            }
+        }
+
+        return analyzed;
+    }
+
+    private static int getPriority(Operator op)
+    {
+        switch (op)
+        {
+            case EQ:
+            case CONTAINS:
+            case CONTAINS_KEY:
+                return 5;
+
+            case GTE:
+            case GT:
+                return 3;
+
+            case LTE:
+            case LT:
+                return 2;
+
+            default:
+                return 0;
+        }
+    }
+
+    /**
+     * Converts expressions into filter tree for query.
+     *
+     * @return a KeyRangeIterator over the index query results
+     */
+    static KeyRangeIterator buildIterator(QueryController controller)
+    {
+        return Node.buildTree(controller.filterOperation()).analyzeTree(controller).rangeIterator(controller);
+    }
+
+    /**
+     * Converts expressions into filter tree (which is currently just a single AND).
+     *
+     * Filter tree allows us to do a couple of important optimizations
+     * namely, group flattening for AND operations (query rewrite), expression bounds checks,
+     * "satisfies by" checks for resulting rows with an early exit.
+     *
+     * @return root of the filter tree.
+     */
+    static FilterTree buildFilter(QueryController controller)
+    {
+        return Node.buildTree(controller.filterOperation()).buildFilter(controller);
+    }
+
+    static abstract class Node
+    {
+        ListMultimap<ColumnMetadata, Expression> expressionMap;
+
+        boolean canFilter()
+        {
+            return (expressionMap != null && !expressionMap.isEmpty()) || !children().isEmpty();
+        }
+
+        List<Node> children()
+        {
+            return Collections.emptyList();
+        }
+
+        void add(Node child)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        RowFilter.Expression expression()
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        abstract void analyze(List<RowFilter.Expression> expressionList, QueryController controller);
+
+        abstract FilterTree filterTree();
+
+        abstract KeyRangeIterator rangeIterator(QueryController controller);
+
+        static Node buildTree(RowFilter filterOperation)
+        {
+            OperatorNode node = new AndNode();
+            for (RowFilter.Expression expression : filterOperation.getExpressions())
+                node.add(buildExpression(expression));
+            return node;
+        }
+
+        static Node buildExpression(RowFilter.Expression expression)
+        {
+            return new ExpressionNode(expression);
+        }
+
+        Node analyzeTree(QueryController controller)
+        {
+            List<RowFilter.Expression> expressionList = new ArrayList<>();
+            doTreeAnalysis(this, expressionList, controller);
+            if (!expressionList.isEmpty())
+                this.analyze(expressionList, controller);
+            return this;
+        }
+
+        void doTreeAnalysis(Node node, List<RowFilter.Expression> expressions, QueryController controller)
+        {
+            if (node.children().isEmpty())
+                expressions.add(node.expression());
+            else
+            {
+                List<RowFilter.Expression> expressionList = new ArrayList<>();
+                for (Node child : node.children())
+                    doTreeAnalysis(child, expressionList, controller);
+                node.analyze(expressionList, controller);
+            }
+        }
+
+        FilterTree buildFilter(QueryController controller)
+        {
+            analyzeTree(controller);
+            FilterTree tree = filterTree();
+            for (Node child : children())
+                if (child.canFilter())
+                    tree.addChild(child.buildFilter(controller));
+            return tree;
+        }
+    }
+
+    static abstract class OperatorNode extends Node
+    {
+        final List<Node> children = new ArrayList<>();
+
+        @Override
+        public List<Node> children()
+        {
+            return children;
+        }
+
+        @Override
+        public void add(Node child)
+        {
+            children.add(child);
+        }
+    }
+
+    static class AndNode extends OperatorNode
+    {
+        @Override
+        public void analyze(List<RowFilter.Expression> expressionList, QueryController controller)
+        {
+            expressionMap = buildIndexExpressions(controller, BooleanOperator.AND, expressionList);
+        }
+
+        @Override
+        FilterTree filterTree()
+        {
+            return new FilterTree(BooleanOperator.AND, expressionMap);
+        }
+
+        @Override
+        KeyRangeIterator rangeIterator(QueryController controller)
+        {
+            KeyRangeIterator.Builder builder = controller.getIndexQueryResults(expressionMap.values());
+            for (Node child : children)
+            {
+                boolean canFilter = child.canFilter();
+                if (canFilter)
+                    builder.add(child.rangeIterator(controller));
+            }
+            return builder.build();
+        }
+    }
+
+    static class ExpressionNode extends Node
+    {
+        final RowFilter.Expression expression;
+
+        @Override
+        public void analyze(List<RowFilter.Expression> expressionList, QueryController controller)
+        {
+            expressionMap = buildIndexExpressions(controller, BooleanOperator.AND, expressionList);
+        }
+
+        @Override
+        FilterTree filterTree()
+        {
+            return new FilterTree(BooleanOperator.AND, expressionMap);
+        }
+
+        public ExpressionNode(RowFilter.Expression expression)
+        {
+            this.expression = expression;
+        }
+
+        @Override
+        public RowFilter.Expression expression()
+        {
+            return expression;
+        }
+
+        @Override
+        KeyRangeIterator rangeIterator(QueryController controller)
+        {
+            assert canFilter() : "Cannot process query with no expressions";
+
+            return controller.getIndexQueryResults(expressionMap.values()).build();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/plan/QueryController.java b/src/java/org/apache/cassandra/index/sai/plan/QueryController.java
new file mode 100644
index 0000000000..0c0e033f5c
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/plan/QueryController.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.plan;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.collect.Lists;
+
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.PartitionRangeReadCommand;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.SinglePartitionReadCommand;
+import org.apache.cassandra.db.filter.ClusteringIndexFilter;
+import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
+import org.apache.cassandra.db.filter.DataLimits;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.metrics.TableQueryMetrics;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.KeyRangeIntersectionIterator;
+import org.apache.cassandra.index.sai.utils.KeyRangeIterator;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.FBUtilities;
+
+public class QueryController
+{
+    private final ColumnFamilyStore cfs;
+    private final ReadCommand command;
+    private final QueryContext queryContext;
+    private final TableQueryMetrics tableQueryMetrics;
+    private final RowFilter filterOperation;
+    private final List<DataRange> ranges;
+    private final AbstractBounds<PartitionPosition> mergeRange;
+
+    public QueryController(ColumnFamilyStore cfs,
+                           ReadCommand command,
+                           RowFilter filterOperation,
+                           QueryContext queryContext,
+                           TableQueryMetrics tableQueryMetrics)
+    {
+        this.cfs = cfs;
+        this.command = command;
+        this.queryContext = queryContext;
+        this.tableQueryMetrics = tableQueryMetrics;
+        this.filterOperation = filterOperation;
+        this.ranges = dataRanges(command);
+        DataRange first = ranges.get(0);
+        DataRange last = ranges.get(ranges.size() - 1);
+        this.mergeRange = ranges.size() == 1 ? first.keyRange() : first.keyRange().withNewRight(last.keyRange().right);
+    }
+
+    public TableMetadata metadata()
+    {
+        return command.metadata();
+    }
+
+    RowFilter filterOperation()
+    {
+        return this.filterOperation;
+    }
+
+    /**
+     * @return token ranges used in the read command
+     */
+    List<DataRange> dataRanges()
+    {
+        return ranges;
+    }
+
+    /**
+     * Note: merged range may contain subrange that no longer belongs to the local node after range movement.
+     * It should only be used as an optimization to reduce search space. Use {@link #dataRanges()} instead to filter data.
+     *
+     * @return merged token range
+     */
+    AbstractBounds<PartitionPosition> mergeRange()
+    {
+        return mergeRange;
+    }
+
+    /**
+     * @return indexed {@code IndexContext} if index is found; otherwise return non-indexed {@code IndexContext}.
+     */
+    public IndexContext getContext(RowFilter.Expression expression)
+    {
+        Set<StorageAttachedIndex> indexes = cfs.indexManager.getBestIndexFor(expression, StorageAttachedIndex.class);
+
+        return indexes.isEmpty() ? new IndexContext(cfs.metadata().keyspace,
+                                                    cfs.metadata().name,
+                                                    cfs.metadata().partitionKeyType,
+                                                    cfs.metadata().comparator,
+                                                    expression.column(),
+                                                    IndexTarget.Type.VALUES,
+                                                    null)
+                                 : indexes.iterator().next().getIndexContext();
+    }
+
+    public UnfilteredRowIterator queryStorage(PrimaryKey key, ReadExecutionController executionController)
+    {
+        if (key == null)
+            throw new IllegalArgumentException("non-null key required");
+
+        try
+        {
+            SinglePartitionReadCommand partition = SinglePartitionReadCommand.create(cfs.metadata(),
+                                                                                     command.nowInSec(),
+                                                                                     command.columnFilter(),
+                                                                                     RowFilter.none(),
+                                                                                     DataLimits.NONE,
+                                                                                     key.partitionKey(),
+                                                                                     makeFilter(key));
+
+            return partition.queryMemtableAndDisk(cfs, executionController);
+        }
+        finally
+        {
+            queryContext.checkpoint();
+        }
+    }
+
+    /**
+     * Build a {@link KeyRangeIterator.Builder} from the given list of expressions by applying given operation (AND).
+     * Building of such builder involves index search, results of which are persisted in the internal resources list
+     *
+     * @param expressions The expressions to build range iterator from (expressions with not results are ignored).
+     *
+     * @return range iterator builder based on given expressions and operation type.
+     */
+    public KeyRangeIterator.Builder getIndexQueryResults(Collection<Expression> expressions)
+    {
+        KeyRangeIterator.Builder builder = KeyRangeIntersectionIterator.builder(expressions.size());
+
+        try
+        {
+            for (Expression e : expressions)
+            {
+                if (e.context.isIndexed())
+                {
+                    @SuppressWarnings({"resource", "RedundantSuppression"}) // RangeIterators are closed by releaseIndexes
+                    KeyRangeIterator memtableIterator = e.context.searchMemtableIndexes(e, mergeRange);
+
+                    builder.add(memtableIterator);
+                }
+            }
+        }
+        catch (Throwable t)
+        {
+            // all sstable indexes in view have been referenced, need to clean up when exception is thrown
+            builder.cleanup();
+            throw t;
+        }
+        return builder;
+    }
+
+    /**
+     * Returns whether this query is not selecting the {@link PrimaryKey}.
+     * The query does not select the key if both of the following statements are false:
+     *  1. The table associated with the query is not using clustering keys
+     *  2. The clustering index filter for the command wants the row.
+     *
+     *  Item 2 is important in paged queries where the {@link org.apache.cassandra.db.filter.ClusteringIndexSliceFilter} for
+     *  subsequent paged queries may not select rows that are returned by the index
+     *  search because that is initially partition based.
+     *
+     * @param key The {@link PrimaryKey} to be tested
+     * @return true if the key is not selected by the query
+     */
+    public boolean doesNotSelect(PrimaryKey key)
+    {
+        return !key.hasEmptyClustering() && !command.clusteringIndexFilter(key.partitionKey()).selects(key.clustering());
+    }
+
+    // Note: This method assumes that the selects method has already been called for the
+    // key to avoid having to (potentially) call selects twice
+    private ClusteringIndexFilter makeFilter(PrimaryKey key)
+    {
+        ClusteringIndexFilter clusteringIndexFilter = command.clusteringIndexFilter(key.partitionKey());
+
+        if (key.hasEmptyClustering())
+            return clusteringIndexFilter;
+        else
+            return new ClusteringIndexNamesFilter(FBUtilities.singleton(key.clustering(), cfs.metadata().comparator),
+                                                  clusteringIndexFilter.isReversed());
+    }
+
+    /**
+     * Used to release all resources and record metrics when query finishes.
+     */
+    public void finish()
+    {
+        if (tableQueryMetrics != null) tableQueryMetrics.record(queryContext);
+    }
+
+    /**
+     * Returns the {@link DataRange} list covered by the specified {@link ReadCommand}.
+     *
+     * @param command a read command
+     * @return the data ranges covered by {@code command}
+     */
+    private static List<DataRange> dataRanges(ReadCommand command)
+    {
+        if (command instanceof SinglePartitionReadCommand)
+        {
+            SinglePartitionReadCommand cmd = (SinglePartitionReadCommand) command;
+            DecoratedKey key = cmd.partitionKey();
+            return Lists.newArrayList(new DataRange(new Range<>(key, key), cmd.clusteringIndexFilter()));
+        }
+        else if (command instanceof PartitionRangeReadCommand)
+        {
+            PartitionRangeReadCommand cmd = (PartitionRangeReadCommand) command;
+            return Lists.newArrayList(cmd.dataRange());
+        }
+        else
+        {
+            throw new AssertionError("Unsupported read command type: " + command.getClass().getName());
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexQueryPlan.java b/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexQueryPlan.java
new file mode 100644
index 0000000000..a2563e638d
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexQueryPlan.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.plan;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
+
+import com.google.common.collect.ImmutableSet;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.metrics.TableQueryMetrics;
+import org.apache.cassandra.schema.TableMetadata;
+
+public class StorageAttachedIndexQueryPlan implements Index.QueryPlan
+{
+    private final ColumnFamilyStore cfs;
+    private final TableQueryMetrics queryMetrics;
+    private final RowFilter postIndexFilter;
+    private final RowFilter filterOperation;
+    private final Set<Index> indexes;
+
+    private StorageAttachedIndexQueryPlan(ColumnFamilyStore cfs,
+                                          TableQueryMetrics queryMetrics,
+                                          RowFilter postIndexFilter,
+                                          RowFilter filterOperation,
+                                          ImmutableSet<Index> indexes)
+    {
+        this.cfs = cfs;
+        this.queryMetrics = queryMetrics;
+        this.postIndexFilter = postIndexFilter;
+        this.filterOperation = filterOperation;
+        this.indexes = indexes;
+    }
+
+    @Nullable
+    public static StorageAttachedIndexQueryPlan create(ColumnFamilyStore cfs,
+                                                       TableQueryMetrics queryMetrics,
+                                                       Set<Index> indexes,
+                                                       RowFilter rowFilter)
+    {
+        ImmutableSet.Builder<Index> selectedIndexesBuilder = ImmutableSet.builder();
+
+        for (RowFilter.Expression expression : rowFilter)
+        {
+            // we ignore user-defined expressions here because we don't have a way to translate their #isSatifiedBy
+            // method, they will be included in the filter returned by QueryPlan#postIndexQueryFilter()
+            if (expression.isUserDefined())
+                continue;
+
+            for (Index index : indexes)
+            {
+                if (index.supportsExpression(expression.column(), expression.operator()))
+                {
+                    selectedIndexesBuilder.add(index);
+                }
+            }
+        }
+
+        ImmutableSet<Index> selectedIndexes = selectedIndexesBuilder.build();
+        if (selectedIndexes.isEmpty())
+            return null;
+
+        /*
+         * postIndexFilter comprised by those expressions in the read command row filter that can't be handled by
+         * {@link FilterTree#satisfiedBy(Unfiltered, Row, boolean)}. This includes expressions targeted
+         * at {@link RowFilter.UserExpression}s.
+         */
+        RowFilter postIndexFilter = rowFilter.restrict(RowFilter.Expression::isUserDefined);
+        return new StorageAttachedIndexQueryPlan(cfs, queryMetrics, postIndexFilter, rowFilter, selectedIndexes);
+    }
+
+    @Override
+    public Set<Index> getIndexes()
+    {
+        return indexes;
+    }
+
+    @Override
+    public long getEstimatedResultRows()
+    {
+        // this is temporary (until proper QueryPlan is integrated into Cassandra)
+        // and allows us to priority storage-attached indexes if any in the query since they
+        // are going to be more efficient, to query and intersect, than built-in indexes.
+        return Long.MIN_VALUE;
+    }
+
+    @Override
+    public boolean shouldEstimateInitialConcurrency()
+    {
+        return false;
+    }
+
+    @Override
+    public Index.Searcher searcherFor(ReadCommand command)
+    {
+        return new StorageAttachedIndexSearcher(cfs,
+                                                queryMetrics,
+                                                command,
+                                                filterOperation,
+                                                DatabaseDescriptor.getRangeRpcTimeout(TimeUnit.MILLISECONDS));
+    }
+
+    /**
+     * @return a filter with all the expressions that are user-defined or for a non-indexed partition key column
+     *
+     * (currently index on partition columns is not supported, see {@link StorageAttachedIndex#validateOptions(Map, TableMetadata)})
+     */
+    @Override
+    public RowFilter postIndexQueryFilter()
+    {
+        return postIndexFilter;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexSearcher.java b/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexSearcher.java
new file mode 100644
index 0000000000..d3860db544
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/plan/StorageAttachedIndexSearcher.java
@@ -0,0 +1,566 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.plan;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.function.Supplier;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import com.google.common.collect.Iterators;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DataRange;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.ReadExecutionController;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+import org.apache.cassandra.db.rows.AbstractUnfilteredRowIterator;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.RowIterator;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.exceptions.RequestTimeoutException;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sai.metrics.TableQueryMetrics;
+import org.apache.cassandra.index.sai.utils.KeyRangeIterator;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyFactory;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.AbstractIterator;
+
+public class StorageAttachedIndexSearcher implements Index.Searcher
+{
+    private final ReadCommand command;
+    private final QueryController queryController;
+    private final QueryContext queryContext;
+    private final PrimaryKeyFactory keyFactory;
+
+    public StorageAttachedIndexSearcher(ColumnFamilyStore cfs,
+                                        TableQueryMetrics tableQueryMetrics,
+                                        ReadCommand command,
+                                        RowFilter filterOperation,
+                                        long executionQuotaMs)
+    {
+        this.command = command;
+        this.queryContext = new QueryContext(command, executionQuotaMs);
+        this.queryController = new QueryController(cfs, command, filterOperation, queryContext, tableQueryMetrics);
+        this.keyFactory = PrimaryKey.factory(cfs.metadata().comparator);
+    }
+
+    @Override
+    public ReadCommand command()
+    {
+        return command;
+    }
+
+    @Override
+    public PartitionIterator filterReplicaFilteringProtection(PartitionIterator fullResponse)
+    {
+        for (RowFilter.Expression expression : queryController.filterOperation())
+        {
+            AbstractAnalyzer analyzer = queryController.getContext(expression).getIndexAnalyzerFactory().create();
+            try
+            {
+                if (analyzer.transformValue())
+                    return applyIndexFilter(fullResponse, Operation.buildFilter(queryController), queryContext);
+            }
+            finally
+            {
+                analyzer.end();
+            }
+        }
+
+        // if no analyzer does transformation
+        return Index.Searcher.super.filterReplicaFilteringProtection(fullResponse);
+    }
+
+    @Override
+    public UnfilteredPartitionIterator search(ReadExecutionController executionController) throws RequestTimeoutException
+    {
+        return new ResultRetriever(queryController, executionController, queryContext, keyFactory);
+    }
+
+    private static class ResultRetriever extends AbstractIterator<UnfilteredRowIterator> implements UnfilteredPartitionIterator
+    {
+        private final PrimaryKey firstPrimaryKey;
+        private final PrimaryKey lastPrimaryKey;
+        private final Iterator<DataRange> keyRanges;
+        private AbstractBounds<PartitionPosition> currentKeyRange;
+
+        private final KeyRangeIterator resultKeyIterator;
+        private final FilterTree filterTree;
+        private final QueryController queryController;
+        private final ReadExecutionController executionController;
+        private final QueryContext queryContext;
+        private final PrimaryKeyFactory keyFactory;
+        private PrimaryKey lastKey;
+
+        private ResultRetriever(QueryController queryController,
+                                ReadExecutionController executionController,
+                                QueryContext queryContext,
+                                PrimaryKeyFactory keyFactory)
+        {
+            this.keyRanges = queryController.dataRanges().iterator();
+            this.currentKeyRange = keyRanges.next().keyRange();
+
+            this.resultKeyIterator = Operation.buildIterator(queryController);
+            this.filterTree = Operation.buildFilter(queryController);
+            this.queryController = queryController;
+            this.executionController = executionController;
+            this.queryContext = queryContext;
+            this.keyFactory = keyFactory;
+
+            this.firstPrimaryKey = keyFactory.createTokenOnly(queryController.mergeRange().left.getToken());
+            this.lastPrimaryKey = keyFactory.createTokenOnly(queryController.mergeRange().right.getToken());
+        }
+
+        @Override
+        @SuppressWarnings({"resource", "RedundantSuppression"}) // The iterator produced here has a nop close operation
+        public UnfilteredRowIterator computeNext()
+        {
+            // IMPORTANT: The correctness of the entire query pipeline relies on the fact that we consume a token
+            // and materialize its keys before moving on to the next token in the flow. This sequence must not be broken
+            // with toList() or similar. (Both the union and intersection flow constructs, to avoid excessive object
+            // allocation, reuse their token mergers as they process individual positions on the ring.)
+
+            if (resultKeyIterator == null)
+                return endOfData();
+
+            // If being called for the first time, skip to the beginning of the range.
+            // We can't put this code in the constructor because it may throw and the caller
+            // may not be prepared for that.
+            if (lastKey == null)
+                resultKeyIterator.skipTo(firstPrimaryKey);
+
+            // Theoretically we wouldn't need this if the caller of computeNext always ran the
+            // returned iterators to the completion. Unfortunately, we have no control over the caller behavior here.
+            // Hence, we skip to the next partition in order to comply to the unwritten partition iterator contract
+            // saying this iterator must not return the same partition twice.
+            skipToNextPartition();
+
+            UnfilteredRowIterator iterator = nextRowIterator(this::nextSelectedKeyInRange);
+            return iterator != null
+                   ? iteratePartition(iterator)
+                   : endOfData();
+        }
+
+        /**
+         * Tries to obtain a row iterator for one of the supplied keys by repeatedly calling
+         * {@link ResultRetriever#queryStorageAndFilter} until it gives a non-null result.
+         * The keySupplier should return the next key with every call to get() and
+         * null when there are no more keys to try.
+         *
+         * @return an iterator or null if all keys were tried with no success
+         */
+        private @Nullable UnfilteredRowIterator nextRowIterator(@Nonnull Supplier<PrimaryKey> keySupplier)
+        {
+            UnfilteredRowIterator iterator = null;
+            while (iterator == null)
+            {
+                PrimaryKey key = keySupplier.get();
+                if (key == null)
+                    return null;
+                iterator = queryStorageAndFilter(key);
+            }
+            return iterator;
+        }
+
+        /**
+         * Returns the next available key contained by one of the keyRanges.
+         * If the next key falls out of the current key range, it skips to the next key range, and so on.
+         * If no more keys or no more ranges are available, returns null.
+         */
+        private @Nullable PrimaryKey nextKeyInRange()
+        {
+            PrimaryKey key = nextKey();
+
+            while (key != null && !(currentKeyRange.contains(key.partitionKey())))
+            {
+                if (!currentKeyRange.right.isMinimum() && currentKeyRange.right.compareTo(key.partitionKey()) <= 0)
+                {
+                    // currentKeyRange before the currentKey so need to move currentKeyRange forward
+                    currentKeyRange = nextKeyRange();
+                    if (currentKeyRange == null)
+                        return null;
+                }
+                else
+                {
+                    // key either before the current range, so let's move the key forward
+                    skipTo(currentKeyRange.left.getToken());
+                    key = nextKey();
+                }
+            }
+            return key;
+        }
+
+        /**
+         * Returns the next available key contained by one of the keyRanges and selected by the queryController.
+         * If the next key falls out of the current key range, it skips to the next key range, and so on.
+         * If no more keys acceptd by the controller are available, returns null.
+         */
+        private @Nullable PrimaryKey nextSelectedKeyInRange()
+        {
+            PrimaryKey key;
+            do
+            {
+                key = nextKeyInRange();
+            }
+            while (key != null && queryController.doesNotSelect(key));
+            return key;
+        }
+
+        /**
+         * Retrieves the next primary key that belongs to the given partition and is selected by the query controller.
+         * The underlying key iterator is advanced only if the key belongs to the same partition.
+         * <p>
+         * Returns null if:
+         * <ul>
+         *   <li>there are no more keys</li>
+         *   <li>the next key is beyond the upper bound</li>
+         *   <li>the next key belongs to a different partition</li>
+         * </ul>
+         * </p>
+         */
+        private @Nullable PrimaryKey nextSelectedKeyInPartition(DecoratedKey partitionKey)
+        {
+            PrimaryKey key;
+            do
+            {
+                if (!resultKeyIterator.hasNext())
+                    return null;
+                if (!resultKeyIterator.peek().partitionKey().equals(partitionKey))
+                    return null;
+
+                key = nextKey();
+            }
+            while (key != null && queryController.doesNotSelect(key));
+            return key;
+        }
+
+        /**
+         * Gets the next key from the underlying operation.
+         * Returns null if there are no more keys <= lastPrimaryKey.
+         */
+        private @Nullable PrimaryKey nextKey()
+        {
+            if (!resultKeyIterator.hasNext())
+                return null;
+            PrimaryKey key = resultKeyIterator.next();
+            return isWithinUpperBound(key) ? key : null;
+        }
+
+        /**
+         * Returns true if the key is not greater than lastPrimaryKey
+         */
+        private boolean isWithinUpperBound(PrimaryKey key)
+        {
+            return lastPrimaryKey.token().isMinimum() || lastPrimaryKey.compareTo(key) >= 0;
+        }
+
+        /**
+         * Gets the next key range from the underlying range iterator.
+         */
+        private @Nullable AbstractBounds<PartitionPosition> nextKeyRange()
+        {
+            return keyRanges.hasNext() ? keyRanges.next().keyRange() : null;
+        }
+
+        /**
+         * Convenience function to skip to a given token.
+         */
+        private void skipTo(@Nonnull Token token)
+        {
+            resultKeyIterator.skipTo(keyFactory.createTokenOnly(token));
+        }
+
+        /**
+         * Skips to the key that belongs to a different partition than the last key we fetched.
+         */
+        private void skipToNextPartition()
+        {
+            if (lastKey == null)
+                return;
+            DecoratedKey lastPartitionKey = lastKey.partitionKey();
+            while (resultKeyIterator.hasNext() && resultKeyIterator.peek().partitionKey().equals(lastPartitionKey))
+                resultKeyIterator.next();
+        }
+
+
+        /**
+         * Returns an iterator over the rows in the partition associated with the given iterator.
+         * Initially, it retrieves the rows from the given iterator until it runs out of data.
+         * Then it iterates the primary keys obtained from the index until the end of the partition
+         * and lazily constructs new row itertors for each of the keys. At a given time, only one row iterator is open.
+         *
+         * The rows are retrieved in the order of primary keys provided by the underlying index.
+         * The iterator is complete when the next key to be fetched belongs to different partition
+         * (but the iterator does not consume that key).
+         *
+         * @param startIter an iterator positioned at the first row in the partition that we want to return
+         */
+        private @Nonnull UnfilteredRowIterator iteratePartition(@Nonnull UnfilteredRowIterator startIter)
+        {
+            return new AbstractUnfilteredRowIterator(startIter.metadata(),
+                                                     startIter.partitionKey(),
+                                                     startIter.partitionLevelDeletion(),
+                                                     startIter.columns(),
+                                                     startIter.staticRow(),
+                                                     startIter.isReverseOrder(),
+                                                     startIter.stats())
+            {
+                private UnfilteredRowIterator currentIter = startIter;
+                private final DecoratedKey partitionKey = startIter.partitionKey();
+
+                @Override
+                protected Unfiltered computeNext()
+                {
+                    while (!currentIter.hasNext())
+                    {
+                        currentIter.close();
+                        currentIter = nextRowIterator(() -> nextSelectedKeyInPartition(partitionKey));
+                        if (currentIter == null)
+                            return endOfData();
+                    }
+                    return currentIter.next();
+                }
+
+                @Override
+                public void close()
+                {
+                    FileUtils.closeQuietly(currentIter);
+                    super.close();
+                }
+            };
+        }
+
+        private UnfilteredRowIterator queryStorageAndFilter(PrimaryKey key)
+        {
+            // Key reads are lazy, delayed all the way to this point. Skip if we've already seen this one:
+            if (key.equals(lastKey))
+                return null;
+
+            lastKey = key;
+
+            try (UnfilteredRowIterator partition = queryController.queryStorage(key, executionController))
+            {
+                queryContext.partitionsRead++;
+
+                return applyIndexFilter(partition, filterTree, queryContext);
+            }
+        }
+
+        private static UnfilteredRowIterator applyIndexFilter(UnfilteredRowIterator partition, FilterTree tree, QueryContext queryContext)
+        {
+            Row staticRow = partition.staticRow();
+            List<Unfiltered> clusters = new ArrayList<>();
+
+            while (partition.hasNext())
+            {
+                Unfiltered row = partition.next();
+
+                queryContext.rowsFiltered++;
+                if (tree.isSatisfiedBy(partition.partitionKey(), row, staticRow))
+                {
+                    clusters.add(row);
+                }
+            }
+
+            if (clusters.isEmpty())
+            {
+                queryContext.rowsFiltered++;
+                if (tree.isSatisfiedBy(partition.partitionKey(), staticRow, staticRow))
+                {
+                    clusters.add(staticRow);
+                }
+            }
+
+            /*
+             * If {@code clusters} is empty, which means either all clustering row and static row pairs failed,
+             *       or static row and static row pair failed. In both cases, we should not return any partition.
+             * If {@code clusters} is not empty, which means either there are some clustering row and static row pairs match the filters,
+             *       or static row and static row pair matches the filters. In both cases, we should return a partition with static row,
+             *       and remove the static row marker from the {@code clusters} for the latter case.
+             */
+            if (clusters.isEmpty())
+            {
+                return null;
+            }
+
+            return new PartitionIterator(partition, staticRow, Iterators.filter(clusters.iterator(), u -> !((Row)u).isStatic()));
+        }
+
+        private static class PartitionIterator extends AbstractUnfilteredRowIterator
+        {
+            private final Iterator<Unfiltered> rows;
+
+            public PartitionIterator(UnfilteredRowIterator partition, Row staticRow, Iterator<Unfiltered> content)
+            {
+                super(partition.metadata(),
+                      partition.partitionKey(),
+                      partition.partitionLevelDeletion(),
+                      partition.columns(),
+                      staticRow,
+                      partition.isReverseOrder(),
+                      partition.stats());
+
+                rows = content;
+            }
+
+            @Override
+            protected Unfiltered computeNext()
+            {
+                return rows.hasNext() ? rows.next() : endOfData();
+            }
+        }
+
+        @Override
+        public TableMetadata metadata()
+        {
+            return queryController.metadata();
+        }
+
+        @Override
+        public void close()
+        {
+            FileUtils.closeQuietly(resultKeyIterator);
+            queryController.finish();
+        }
+    }
+
+    /**
+     * Used by {@link StorageAttachedIndexSearcher#filterReplicaFilteringProtection} to filter rows for columns that
+     * have transformations so won't get handled correctly by the row filter.
+     */
+    @SuppressWarnings({"resource", "RedundantSuppression"})
+    private static PartitionIterator applyIndexFilter(PartitionIterator response, FilterTree tree, QueryContext queryContext)
+    {
+        return new PartitionIterator()
+        {
+            @Override
+            public void close()
+            {
+                response.close();
+            }
+
+            @Override
+            public boolean hasNext()
+            {
+                return response.hasNext();
+            }
+
+            @Override
+            public RowIterator next()
+            {
+                RowIterator delegate = response.next();
+                Row staticRow = delegate.staticRow();
+
+                return new RowIterator()
+                {
+                    Row next;
+
+                    @Override
+                    public TableMetadata metadata()
+                    {
+                        return delegate.metadata();
+                    }
+
+                    @Override
+                    public boolean isReverseOrder()
+                    {
+                        return delegate.isReverseOrder();
+                    }
+
+                    @Override
+                    public RegularAndStaticColumns columns()
+                    {
+                        return delegate.columns();
+                    }
+
+                    @Override
+                    public DecoratedKey partitionKey()
+                    {
+                        return delegate.partitionKey();
+                    }
+
+                    @Override
+                    public Row staticRow()
+                    {
+                        return staticRow;
+                    }
+
+                    @Override
+                    public void close()
+                    {
+                        delegate.close();
+                    }
+
+                    private Row computeNext()
+                    {
+                        while (delegate.hasNext())
+                        {
+                            Row row = delegate.next();
+                            queryContext.rowsFiltered++;
+                            if (tree.isSatisfiedBy(delegate.partitionKey(), row, staticRow))
+                                return row;
+                        }
+                        return null;
+                    }
+
+                    private Row loadNext()
+                    {
+                        if (next == null)
+                            next = computeNext();
+                        return next;
+                    }
+
+                    @Override
+                    public boolean hasNext()
+                    {
+                        return loadNext() != null;
+                    }
+
+                    @Override
+                    public Row next()
+                    {
+                        Row result = loadNext();
+                        next = null;
+
+                        if (result == null)
+                            throw new NoSuchElementException();
+
+                        return result;
+                    }
+                };
+            }
+        };
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/KeyRangeConcatIterator.java b/src/java/org/apache/cassandra/index/sai/utils/KeyRangeConcatIterator.java
new file mode 100644
index 0000000000..eced43e1ef
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/KeyRangeConcatIterator.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.utils;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.PriorityQueue;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.io.util.FileUtils;
+
+/**
+ * {@link KeyRangeConcatIterator} takes a list of sorted range iterator and concatenates them, leaving duplicates in
+ * place, to produce a new stably sorted iterator. Duplicates are eliminated later in
+ * {@link org.apache.cassandra.index.sai.plan.StorageAttachedIndexSearcher}
+ * as results from multiple SSTable indexes and their respective segments are consumed.
+ *
+ * ex. (1, 2, 3) + (3, 3, 4, 5) -> (1, 2, 3, 3, 3, 4, 5)
+ * ex. (1, 2, 2, 3) + (3, 4, 4, 6, 6, 7) -> (1, 2, 2, 3, 3, 4, 4, 6, 6, 7)
+ *
+ * TODO Investigate removing the use of PriorityQueue from this class <a href="https://issues.apache.org/jira/browse/CASSANDRA-18165">CASSANDRA-18165</a>
+ */
+public class KeyRangeConcatIterator extends KeyRangeIterator
+{
+    private final PriorityQueue<KeyRangeIterator> ranges;
+    private final List<KeyRangeIterator> toRelease;
+
+    protected KeyRangeConcatIterator(KeyRangeIterator.Builder.Statistics statistics, PriorityQueue<KeyRangeIterator> ranges)
+    {
+        super(statistics);
+
+        this.ranges = ranges;
+        this.toRelease = new ArrayList<>(ranges);
+    }
+
+    @Override
+    @SuppressWarnings({"resource", "RedundantSuppression"})
+    protected void performSkipTo(PrimaryKey nextKey)
+    {
+        while (!ranges.isEmpty())
+        {
+            if (ranges.peek().getCurrent().compareTo(nextKey) >= 0)
+                break;
+
+            KeyRangeIterator head = ranges.poll();
+
+            if (head.getMaximum().compareTo(nextKey) >= 0)
+            {
+                head.skipTo(nextKey);
+                ranges.add(head);
+                break;
+            }
+        }
+    }
+
+    @Override
+    public void close()
+    {
+        // due to lazy key fetching, we cannot close iterator immediately
+        FileUtils.closeQuietly(toRelease);
+    }
+
+    @Override
+    @SuppressWarnings({"resource", "RedundantSuppression"})
+    protected PrimaryKey computeNext()
+    {
+        while (!ranges.isEmpty())
+        {
+            KeyRangeIterator current = ranges.poll();
+            if (current.hasNext())
+            {
+                PrimaryKey next = current.next();
+                // hasNext will update RangeIterator's current which is used to sort in PQ
+                if (current.hasNext())
+                    ranges.add(current);
+
+                return next;
+            }
+        }
+
+        return endOfData();
+    }
+
+    public static Builder builder(int size)
+    {
+        return new Builder(size);
+    }
+
+    public static KeyRangeIterator build(List<KeyRangeIterator> keys)
+    {
+        return new Builder(keys.size()).add(keys).build();
+    }
+
+    @VisibleForTesting
+    public static class Builder extends KeyRangeIterator.Builder
+    {
+        private final PriorityQueue<KeyRangeIterator> ranges;
+
+        Builder(int size)
+        {
+            super(new ConcatStatistics());
+            ranges = new PriorityQueue<>(size, Comparator.comparing(KeyRangeIterator::getCurrent));
+        }
+
+        @Override
+        public KeyRangeIterator.Builder add(KeyRangeIterator range)
+        {
+            if (range == null)
+                return this;
+
+            if (range.getCount() > 0)
+                ranges.add(range);
+            else
+                FileUtils.closeQuietly(range);
+            statistics.update(range);
+
+            return this;
+        }
+
+        @Override
+        public int rangeCount()
+        {
+            return ranges.size();
+        }
+
+        @Override
+        public void cleanup()
+        {
+            FileUtils.closeQuietly(ranges);
+        }
+
+        @Override
+        protected KeyRangeIterator buildIterator()
+        {
+            if (rangeCount() == 1)
+                return ranges.poll();
+
+            return new KeyRangeConcatIterator(statistics, ranges);
+        }
+    }
+
+    private static class ConcatStatistics extends KeyRangeIterator.Builder.Statistics
+    {
+        @Override
+        public void update(KeyRangeIterator range)
+        {
+            // range iterators should be sorted, but previous max must not be greater than next min.
+            if (range.getCount() > 0)
+            {
+                if (count == 0)
+                {
+                    min = range.getMinimum();
+                }
+                else if (count > 0 && max.compareTo(range.getMinimum()) > 0)
+                {
+                    throw new IllegalArgumentException("RangeIterator must be sorted, previous max: " + max + ", next min: " + range.getMinimum());
+                }
+
+                max = range.getMaximum();
+                count += range.getCount();
+            }
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/KeyRangeIntersectionIterator.java b/src/java/org/apache/cassandra/index/sai/utils/KeyRangeIntersectionIterator.java
new file mode 100644
index 0000000000..2b309095d1
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/KeyRangeIntersectionIterator.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.utils;
+
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.tracing.Tracing;
+
+/**
+ * A simple intersection iterator that makes no real attempts at optimising the iteration apart from
+ * initially sorting the ranges. This implementation also supports an intersection limit which limits
+ * the number of ranges that will be included in the intersection. This currently defaults to 2.
+ */
+@SuppressWarnings({"resource", "RedundantSuppression"})
+public class KeyRangeIntersectionIterator extends KeyRangeIterator
+{
+    private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+    // The cassandra.sai.intersection.clause.limit (default: 2) controls the maximum number of range iterator that
+    // will be used in the final intersection of a query operation.
+    private static final int INTERSECTION_CLAUSE_LIMIT = CassandraRelevantProperties.SAI_INTERSECTION_CLAUSE_LIMIT.getInt();
+
+    static
+    {
+        logger.info(String.format("Storage attached index intersection clause limit is %d", INTERSECTION_CLAUSE_LIMIT));
+    }
+
+    private final List<KeyRangeIterator> ranges;
+
+    private KeyRangeIntersectionIterator(Builder.Statistics statistics, List<KeyRangeIterator> ranges)
+    {
+        super(statistics);
+        this.ranges = ranges;
+    }
+
+    @Override
+    protected PrimaryKey computeNext()
+    {
+        // Range iterator that has been advanced in the previous cycle of the outer loop.
+        // Initially there hasn't been the previous cycle, so set to null.
+        int alreadyAvanced = -1;
+
+        // The highest primary key seen on any range iterator so far.
+        // It can become null when we reach the end of the iterator.
+        PrimaryKey highestKey = getCurrent();
+
+        outer:
+        // We need to check if highestKey exceeds the maximum because the maximum is
+        // the lowest value maximum of all the ranges. As a result any value could
+        // potentially exceed it.
+        while (highestKey != null && highestKey.compareTo(getMaximum()) <= 0)
+        {
+            // Try advance all iterators to the highest key seen so far.
+            // Once this inner loop finishes normally, all iterators are guaranteed to be at the same value.
+            for (int index = 0; index < ranges.size(); index++)
+            {
+                if (index != alreadyAvanced)
+                {
+                    KeyRangeIterator range = ranges.get(index);
+                    PrimaryKey nextKey = nextOrNull(range, highestKey);
+                    if (nextKey == null || nextKey.compareTo(highestKey) > 0)
+                    {
+                        // We jumped over the highest key seen so far, so make it the new highest key.
+                        highestKey = nextKey;
+                        // Remember this iterator to avoid advancing it again, because it is already at the highest key
+                        alreadyAvanced = index;
+                        // This iterator jumped over, so the other iterators are lagging behind now,
+                        // including the ones already advanced in the earlier cycles of the inner loop.
+                        // Therefore, restart the inner loop in order to advance
+                        // the other iterators except this one to match the new highest key.
+                        continue outer;
+                    }
+                    assert nextKey.compareTo(highestKey) == 0:
+                    String.format("skipped to an item smaller than the target; " +
+                                  "iterator: %s, target key: %s, returned key: %s", range, highestKey, nextKey);
+                }
+            }
+            // If we reached here, next() has been called at least once on each range iterator and
+            // the last call to next() on each iterator returned a value equal to the highestKey.
+            return highestKey;
+        }
+        return endOfData();
+    }
+
+    @Override
+    protected void performSkipTo(PrimaryKey nextKey)
+    {
+        for (KeyRangeIterator range : ranges)
+            if (range.hasNext())
+                range.skipTo(nextKey);
+    }
+
+    @Override
+    public void close()
+    {
+        FileUtils.closeQuietly(ranges);
+    }
+
+    /**
+     * Fetches the next available item from the iterator, such that the item is not lower than the given key.
+     * If no such items are available, returns null.
+     */
+    private PrimaryKey nextOrNull(KeyRangeIterator iterator, PrimaryKey minKey)
+    {
+        iterator.skipTo(minKey);
+        return iterator.hasNext() ? iterator.next() : null;
+    }
+
+    public static Builder builder(int size)
+    {
+        return builder(size, INTERSECTION_CLAUSE_LIMIT);
+    }
+
+    @VisibleForTesting
+    public static Builder builder(int size, int limit)
+    {
+        return new Builder(size, limit);
+    }
+
+    @VisibleForTesting
+    public static class Builder extends KeyRangeIterator.Builder
+    {
+        private final int limit;
+        // tracks if any of the added ranges are disjoint with the other ranges, which is useful
+        // in case of intersection, as it gives a direct answer whether the iterator is going
+        // to produce any results.
+        private boolean isDisjoint;
+
+        protected final List<KeyRangeIterator> rangeIterators;
+
+        Builder(int size, int limit)
+        {
+            super(new IntersectionStatistics());
+            rangeIterators = new ArrayList<>(size);
+            this.limit = limit;
+        }
+
+        @Override
+        public KeyRangeIterator.Builder add(KeyRangeIterator range)
+        {
+            if (range == null)
+                return this;
+
+            if (range.getCount() > 0)
+                rangeIterators.add(range);
+            else
+                FileUtils.closeQuietly(range);
+
+            updateStatistics(statistics, range);
+
+            return this;
+        }
+
+        @Override
+        public int rangeCount()
+        {
+            return rangeIterators.size();
+        }
+
+        @Override
+        public void cleanup()
+        {
+            FileUtils.closeQuietly(rangeIterators);
+        }
+
+        @Override
+        protected KeyRangeIterator buildIterator()
+        {
+            rangeIterators.sort(Comparator.comparingLong(KeyRangeIterator::getCount));
+            int initialSize = rangeIterators.size();
+            // all ranges will be included
+            if (limit >= rangeIterators.size() || limit <= 0)
+                return buildIterator(statistics, rangeIterators);
+
+            // Apply most selective iterators during intersection, because larger number of iterators will result lots of disk seek.
+            Statistics selectiveStatistics = new IntersectionStatistics();
+            isDisjoint = false;
+            for (int i = rangeIterators.size() - 1; i >= 0 && i >= limit; i--)
+                FileUtils.closeQuietly(rangeIterators.remove(i));
+
+            rangeIterators.forEach(range -> updateStatistics(selectiveStatistics, range));
+
+            if (Tracing.isTracing())
+                Tracing.trace("Selecting {} {} of {} out of {} indexes",
+                              rangeIterators.size(),
+                              rangeIterators.size() > 1 ? "indexes with cardinalities" : "index with cardinality",
+                              rangeIterators.stream().map(KeyRangeIterator::getCount).map(Object::toString).collect(Collectors.joining(", ")),
+                              initialSize);
+
+            return buildIterator(selectiveStatistics, rangeIterators);
+        }
+
+        public boolean isDisjoint()
+        {
+            return isDisjoint;
+        }
+
+        private KeyRangeIterator buildIterator(Statistics statistics, List<KeyRangeIterator> ranges)
+        {
+            // if the ranges are disjoint, or we have an intersection with an empty set,
+            // we can simply return an empty iterator, because it's not going to produce any results.
+            if (isDisjoint)
+            {
+                FileUtils.closeQuietly(ranges);
+                return KeyRangeIterator.empty();
+            }
+
+            if (ranges.size() == 1)
+                return ranges.get(0);
+
+            return new KeyRangeIntersectionIterator(statistics, ranges);
+        }
+
+        private void updateStatistics(Statistics statistics, KeyRangeIterator range)
+        {
+            statistics.update(range);
+            isDisjoint |= isDisjointInternal(statistics.min, statistics.max, range);
+        }
+    }
+
+    private static class IntersectionStatistics extends KeyRangeIterator.Builder.Statistics
+    {
+        @Override
+        public void update(KeyRangeIterator range)
+        {
+            // minimum of the intersection is the biggest minimum of individual iterators
+            min = nullSafeMax(min, range.getMinimum());
+            // maximum of the intersection is the smallest maximum of individual iterators
+            max = nullSafeMin(max, range.getMaximum());
+            count += range.getCount();
+        }
+    }
+
+    @VisibleForTesting
+    protected static boolean isDisjoint(KeyRangeIterator a, KeyRangeIterator b)
+    {
+        return isDisjointInternal(a.getCurrent(), a.getMaximum(), b);
+    }
+
+    /**
+     * Ranges are overlapping the following cases:
+     *
+     *   * When they have a common subrange:
+     *
+     *   min       b.current      max          b.max
+     *   +---------|--------------+------------|
+     *
+     *   b.current      min       max          b.max
+     *   |--------------+---------+------------|
+     *
+     *   min        b.current     b.max        max
+     *   +----------|-------------|------------+
+     *
+     *
+     *  If either range is empty, they're disjoint.
+     */
+    private static boolean isDisjointInternal(PrimaryKey min, PrimaryKey max, KeyRangeIterator b)
+    {
+        return min == null || max == null || b.getCount() == 0 || min.compareTo(b.getMaximum()) > 0 || b.getCurrent().compareTo(max) > 0;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/KeyRangeIterator.java b/src/java/org/apache/cassandra/index/sai/utils/KeyRangeIterator.java
new file mode 100644
index 0000000000..ff43dbf26a
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/KeyRangeIterator.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.utils;
+
+import java.io.Closeable;
+import java.util.List;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.utils.AbstractGuavaIterator;
+
+/**
+ * An abstract implementation of {@link AbstractGuavaIterator} that supports the building and management of
+ * concatanation, union and intersection iterators.
+ */
+public abstract class KeyRangeIterator extends AbstractGuavaIterator<PrimaryKey> implements Closeable
+{
+    private final PrimaryKey min, max;
+    private final long count;
+    private PrimaryKey current;
+
+    protected KeyRangeIterator(Builder.Statistics statistics)
+    {
+        this(statistics.min, statistics.max, statistics.count);
+    }
+
+    public KeyRangeIterator(PrimaryKey min, PrimaryKey max, long count)
+    {
+        boolean isComplete = min != null && max != null && count != 0;
+        boolean isEmpty = min == null && max == null && (count == 0 || count == -1);
+        Preconditions.checkArgument(isComplete || isEmpty, "Range: [" + min + ',' + max + "], Count: " + count);
+
+        this.min = min;
+        this.current = min;
+        this.max = max;
+        this.count = count;
+    }
+
+    public final PrimaryKey getMinimum()
+    {
+        return min;
+    }
+
+    public final PrimaryKey getCurrent()
+    {
+        return current;
+    }
+
+    public final PrimaryKey getMaximum()
+    {
+        return max;
+    }
+
+    public final long getCount()
+    {
+        return count;
+    }
+
+    /**
+     * When called, this iterators current position should
+     * be skipped forwards until finding either:
+     *   1) an element equal to or bigger than next
+     *   2) the end of the iterator
+     *
+     * @param nextKey value to skip the iterator forward until matching
+     *
+     * @return The next current key after the skip was performed
+     */
+    public final PrimaryKey skipTo(PrimaryKey nextKey)
+    {
+        if (min == null || max == null)
+            return endOfData();
+
+        // In the case of deferred iterators the current value may not accurately
+        // reflect the next value, so we need to check that as well
+        if (current.compareTo(nextKey) >= 0)
+        {
+            next = next == null ? recomputeNext() : next;
+            if (next == null)
+                return endOfData();
+            else if (next.compareTo(nextKey) >= 0)
+                return next;
+        }
+
+        if (max.compareTo(nextKey) < 0)
+            return endOfData();
+
+        performSkipTo(nextKey);
+        return recomputeNext();
+    }
+
+    protected abstract void performSkipTo(PrimaryKey nextKey);
+
+    protected PrimaryKey recomputeNext()
+    {
+        return tryToComputeNext() ? peek() : endOfData();
+    }
+
+    protected boolean tryToComputeNext()
+    {
+        boolean hasNext = super.tryToComputeNext();
+        current = hasNext ? next : getMaximum();
+        return hasNext;
+    }
+
+    public static KeyRangeIterator empty()
+    {
+        return EmptyRangeIterator.instance;
+    }
+
+    private static class EmptyRangeIterator extends KeyRangeIterator
+    {
+        static final KeyRangeIterator instance = new EmptyRangeIterator();
+        EmptyRangeIterator() { super(null, null, 0); }
+        public PrimaryKey computeNext() { return endOfData(); }
+        protected void performSkipTo(PrimaryKey nextKey) { }
+        public void close() { }
+    }
+
+    @VisibleForTesting
+    public static abstract class Builder
+    {
+        protected final Statistics statistics;
+
+        public Builder(Statistics statistics)
+        {
+            this.statistics = statistics;
+        }
+
+        public PrimaryKey getMinimum()
+        {
+            return statistics.min;
+        }
+
+        public PrimaryKey getMaximum()
+        {
+            return statistics.max;
+        }
+
+        public long getCount()
+        {
+            return statistics.count;
+        }
+
+        public Builder add(List<KeyRangeIterator> ranges)
+        {
+            if (ranges == null || ranges.isEmpty())
+                return this;
+
+            ranges.forEach(this::add);
+            return this;
+        }
+
+        public final KeyRangeIterator build()
+        {
+            if (rangeCount() == 0)
+                return empty();
+            else
+                return buildIterator();
+        }
+
+        public abstract Builder add(KeyRangeIterator range);
+
+        public abstract int rangeCount();
+
+        public abstract void cleanup();
+
+        protected abstract KeyRangeIterator buildIterator();
+
+        public static abstract class Statistics
+        {
+            protected PrimaryKey min, max;
+            protected long count;
+
+            public abstract void update(KeyRangeIterator range);
+        }
+    }
+
+    protected static <T extends Comparable<T>> T nullSafeMin(T a, T b)
+    {
+        if (a == null) return b;
+        if (b == null) return a;
+
+        return a.compareTo(b) > 0 ? b : a;
+    }
+
+    protected static <T extends Comparable<T>> T nullSafeMax(T a, T b)
+    {
+        if (a == null) return b;
+        if (b == null) return a;
+
+        return a.compareTo(b) > 0 ? a : b;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/KeyRangeUnionIterator.java b/src/java/org/apache/cassandra/index/sai/utils/KeyRangeUnionIterator.java
new file mode 100644
index 0000000000..96bd3432a9
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/KeyRangeUnionIterator.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.utils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.io.util.FileUtils;
+
+/**
+ * Range Union Iterator is used to return sorted stream of elements from multiple RangeIterator instances.
+ */
+@SuppressWarnings({"resource", "RedundantSuppression"})
+public class KeyRangeUnionIterator extends KeyRangeIterator
+{
+    private final List<KeyRangeIterator> ranges;
+    private final List<KeyRangeIterator> candidates;
+
+    private KeyRangeUnionIterator(Builder.Statistics statistics, List<KeyRangeIterator> ranges)
+    {
+        super(statistics);
+        this.ranges = ranges;
+        this.candidates = new ArrayList<>(ranges.size());
+    }
+
+    @Override
+    public PrimaryKey computeNext()
+    {
+        candidates.clear();
+        PrimaryKey candidate = null;
+        for (KeyRangeIterator range : ranges)
+        {
+            if (range.hasNext())
+            {
+                // Avoid repeated values but only if we have read at least one value
+                while (next != null && range.hasNext() && range.peek().compareTo(getCurrent()) == 0)
+                    range.next();
+                if (!range.hasNext())
+                    continue;
+                if (candidate == null)
+                {
+                    candidate = range.peek();
+                    candidates.add(range);
+                }
+                else
+                {
+                    int cmp = candidate.compareTo(range.peek());
+                    if (cmp == 0)
+                        candidates.add(range);
+                    else if (cmp > 0)
+                    {
+                        candidates.clear();
+                        candidate = range.peek();
+                        candidates.add(range);
+                    }
+                }
+            }
+        }
+        if (candidates.isEmpty())
+            return endOfData();
+        candidates.forEach(KeyRangeIterator::next);
+        return candidate;
+    }
+
+    @Override
+    protected void performSkipTo(PrimaryKey nextKey)
+    {
+        for (KeyRangeIterator range : ranges)
+        {
+            if (range.hasNext())
+                range.skipTo(nextKey);
+        }
+    }
+
+    @Override
+    public void close()
+    {
+        // Due to lazy key fetching, we cannot close iterator immediately
+        FileUtils.closeQuietly(ranges);
+    }
+
+    public static Builder builder(int size)
+    {
+        return new Builder(size);
+    }
+
+    public static KeyRangeIterator build(List<KeyRangeIterator> keys)
+    {
+        return new Builder(keys.size()).add(keys).build();
+    }
+
+    @VisibleForTesting
+    public static class Builder extends KeyRangeIterator.Builder
+    {
+        protected final List<KeyRangeIterator> rangeIterators;
+
+        Builder(int size)
+        {
+            super(new UnionStatistics());
+            this.rangeIterators = new ArrayList<>(size);
+        }
+
+        @Override
+        public KeyRangeIterator.Builder add(KeyRangeIterator range)
+        {
+            if (range == null)
+                return this;
+
+            if (range.getCount() > 0)
+            {
+                rangeIterators.add(range);
+                statistics.update(range);
+            }
+            else
+            {
+                FileUtils.closeQuietly(range);
+            }
+
+            return this;
+        }
+
+        @Override
+        public int rangeCount()
+        {
+            return rangeIterators.size();
+        }
+
+        @Override
+        public void cleanup()
+        {
+            FileUtils.closeQuietly(rangeIterators);
+        }
+
+        @Override
+        protected KeyRangeIterator buildIterator()
+        {
+            if (rangeCount() == 1)
+                return rangeIterators.get(0);
+
+            return new KeyRangeUnionIterator(statistics, rangeIterators);
+        }
+    }
+
+    private static class UnionStatistics extends KeyRangeIterator.Builder.Statistics
+    {
+        @Override
+        public void update(KeyRangeIterator range)
+        {
+            min = nullSafeMin(min, range.getMinimum());
+            max = nullSafeMax(max, range.getMaximum());
+            count += range.getCount();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/PrimaryKey.java b/src/java/org/apache/cassandra/index/sai/utils/PrimaryKey.java
new file mode 100644
index 0000000000..038b0b031a
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/PrimaryKey.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.utils;
+
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+
+/**
+ * Representation of the primary key for a row consisting of the {@link DecoratedKey} and
+ * {@link Clustering} associated with a {@link org.apache.cassandra.db.rows.Row}.
+ */
+public class PrimaryKey implements Comparable<PrimaryKey>
+{
+    private final Token token;
+    private final DecoratedKey partitionKey;
+    private final Clustering<?> clustering;
+    private final ClusteringComparator clusteringComparator;
+
+    PrimaryKey(Token token)
+    {
+        this(token, null, null, null);
+    }
+
+    PrimaryKey(Token token,
+               DecoratedKey partitionKey,
+               Clustering<?> clustering,
+               ClusteringComparator clusteringComparator)
+    {
+        this.token = token;
+        this.partitionKey = partitionKey;
+        this.clustering = clustering;
+        this.clusteringComparator = clusteringComparator;
+    }
+
+    /**
+     * Returns a {@link PrimaryKeyFactory} for creating {@link PrimaryKey} instances.
+     *
+     * @param clusteringComparator the {@link ClusteringComparator} used by the
+     *                             {@link PrimaryKeyFactory} for clustering comparisons
+     * @return a {@link PrimaryKeyFactory} for {@link PrimaryKey} creation
+     */
+    public static PrimaryKeyFactory factory(ClusteringComparator clusteringComparator)
+    {
+        return new PrimaryKeyFactory(clusteringComparator);
+    }
+
+    /**
+     * @return the {@link Token} associated with this primary key.
+     */
+    public Token token()
+    {
+        return token;
+    }
+
+    /**
+     * @return the {@link DecoratedKey} associated with this primary key.
+     */
+    public DecoratedKey partitionKey()
+    {
+        return partitionKey;
+    }
+
+    /**
+     * @return the {@link Clustering} associated with this primary key.
+     */
+    public Clustering<?> clustering()
+    {
+        return clustering;
+    }
+
+    /**
+     * Returns the {@link PrimaryKey} as a {@link ByteSource} byte comparable representation.
+     *
+     * It is important that these representations are only ever used with byte comparables using
+     * the same elements. This means that {@code asComparableBytes} responses can only be used
+     * together from the same {@link PrimaryKey} implementation.
+     *
+     * @param version the {@link ByteComparable.Version} to use for the implementation
+     * @return the {@code ByteSource} byte comparable.
+     */
+    public ByteSource asComparableBytes(ByteComparable.Version version)
+    {
+        ByteSource tokenComparable = token.asComparableBytes(version);
+        if (partitionKey == null)
+            return ByteSource.withTerminator(version == ByteComparable.Version.LEGACY ? ByteSource.END_OF_STREAM
+                                                                                      : ByteSource.TERMINATOR,
+                                             tokenComparable,
+                                             null,
+                                             null);
+        ByteSource keyComparable = ByteSource.of(partitionKey.getKey(), version);
+        // It is important that the ClusteringComparator.asBytesComparable method is used
+        // to maintain the correct clustering sort order
+        ByteSource clusteringComparable = clusteringComparator.size() == 0 ||
+                                          clustering == null ||
+                                          clustering.isEmpty() ? null
+                                                               : clusteringComparator.asByteComparable(clustering)
+                                                                                     .asComparableBytes(version);
+        return ByteSource.withTerminator(version == ByteComparable.Version.LEGACY ? ByteSource.END_OF_STREAM
+                                                                                  : ByteSource.TERMINATOR,
+                                         tokenComparable,
+                                         keyComparable,
+                                         clusteringComparable);
+    }
+
+    @Override
+    public int compareTo(PrimaryKey o)
+    {
+        int cmp = token().compareTo(o.token());
+
+        // If the tokens don't match then we don't need to compare any more of the key.
+        // Otherwise, if it's partition key is null or the other partition key is null
+        // then one or both of the keys are token only so we can only compare tokens
+        if ((cmp != 0) || (partitionKey == null) || o.partitionKey() == null)
+            return cmp;
+
+        // Next compare the partition keys. If they are not equal or
+        // this is a single row partition key or there are no
+        // clusterings then we can return the result of this without
+        // needing to compare the clusterings
+        cmp = partitionKey().compareTo(o.partitionKey());
+        if (cmp != 0 || hasEmptyClustering() || o.hasEmptyClustering())
+            return cmp;
+        return clusteringComparator.compare(clustering(), o.clustering());
+    }
+
+    /**
+     * Return whether the primary key has an empty clustering or not.
+     * By default, the clustering is empty if the internal clustering
+     * is null or is empty.
+     *
+     * @return {@code true} if the clustering is empty, otherwise {@code false}
+     */
+    public boolean hasEmptyClustering()
+    {
+        return clustering() == null || clustering().isEmpty();
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(token, partitionKey, clustering, clusteringComparator);
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (obj instanceof PrimaryKey)
+            return compareTo((PrimaryKey)obj) == 0;
+        return false;
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("PrimaryKey: { token: %s, partition: %s, clustering: %s:%s} ",
+                             token,
+                             partitionKey,
+                             clustering == null ? null : clustering.kind(),
+                             clustering == null ? null : Arrays.stream(clustering.getBufferArray())
+                                                               .map(ByteBufferUtil::bytesToHex)
+                                                               .collect(Collectors.joining(",")));
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/PrimaryKeyFactory.java b/src/java/org/apache/cassandra/index/sai/utils/PrimaryKeyFactory.java
new file mode 100644
index 0000000000..f831492ee5
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/PrimaryKeyFactory.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.dht.Token;
+
+/**
+ * A factory for creating instances that are sortable by {@link DecoratedKey} and {@link Clustering}.
+ */
+public class PrimaryKeyFactory
+{
+    private final ClusteringComparator clusteringComparator;
+
+    public PrimaryKeyFactory(ClusteringComparator clusteringComparator)
+    {
+        this.clusteringComparator = clusteringComparator;
+    }
+
+    /**
+     * Creates a {@link PrimaryKey} that is represented by a {@link Token}.
+     *
+     * {@link Token} only primary keys are used for defining the partition range
+     * of a query.
+     */
+    public PrimaryKey createTokenOnly(Token token)
+    {
+        return new PrimaryKey(token);
+    }
+
+    /**
+     * Creates a {@link PrimaryKey} that is fully represented by partition key
+     * and clustering.
+     */
+    public PrimaryKey create(DecoratedKey partitionKey, Clustering<?> clustering)
+    {
+        return new PrimaryKey(partitionKey.getToken(), partitionKey, clustering, clusteringComparator);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/PrimaryKeys.java b/src/java/org/apache/cassandra/index/sai/utils/PrimaryKeys.java
new file mode 100644
index 0000000000..3ba7c08af0
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/PrimaryKeys.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.utils;
+
+import java.util.Iterator;
+import java.util.SortedSet;
+import java.util.concurrent.ConcurrentSkipListSet;
+
+import javax.annotation.concurrent.ThreadSafe;
+
+import org.apache.cassandra.utils.ObjectSizes;
+
+/**
+ * A sorted set of {@link PrimaryKey}s.
+ *
+ * The primary keys are sorted first by token, then by partition key value, and then by clustering.
+ */
+@ThreadSafe
+public class PrimaryKeys implements Iterable<PrimaryKey>
+{
+    private static final long EMPTY_SIZE = ObjectSizes.measure(new PrimaryKeys());
+    // from https://github.com/gaul/java-collection-overhead
+    private static final long SET_ENTRY_OVERHEAD = 36;
+
+    private final ConcurrentSkipListSet<PrimaryKey> keys = new ConcurrentSkipListSet<>();
+
+    /**
+     * Adds a {@link PrimaryKey} and returns the on-heap memory used if the key was added
+     */
+    public long add(PrimaryKey key)
+    {
+        return keys.add(key) ? SET_ENTRY_OVERHEAD : 0;
+    }
+
+    public SortedSet<PrimaryKey> keys()
+    {
+        return keys;
+    }
+
+    public int size()
+    {
+        return keys.size();
+    }
+
+    public boolean isEmpty()
+    {
+        return keys.isEmpty();
+    }
+
+    public long unsharedHeapSize()
+    {
+        return EMPTY_SIZE;
+    }
+
+    @Override
+    public Iterator<PrimaryKey> iterator()
+    {
+        return keys.iterator();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/utils/TypeUtil.java b/src/java/org/apache/cassandra/index/sai/utils/TypeUtil.java
new file mode 100644
index 0000000000..a648c0a8fa
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/utils/TypeUtil.java
@@ -0,0 +1,472 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.utils;
+
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import com.googlecode.concurrenttrees.radix.ConcurrentRadixTree;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.CompositeType;
+import org.apache.cassandra.db.marshal.DecimalType;
+import org.apache.cassandra.db.marshal.InetAddressType;
+import org.apache.cassandra.db.marshal.IntegerType;
+import org.apache.cassandra.db.marshal.ReversedType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.ComplexColumnData;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.FastByteOperations;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+
+public class TypeUtil
+{
+    private static final byte[] IPV4_PREFIX = new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1 };
+
+    /**
+     * DecimalType / BigDecimal values are indexed by truncating their asComparableBytes representation to this size,
+     * padding on the right with zero-value-bytes until this size is reached (if necessary).  This causes
+     * false-positives that must be filtered in a separate step after hitting the index and reading the associated
+     * (full) values.
+     */
+    public static final int DECIMAL_APPROXIMATION_BYTES = 24;
+
+    private TypeUtil() {}
+
+    /**
+     * Returns <code>true</code> if given buffer would pass the {@link AbstractType#validate(ByteBuffer)}
+     * check. False otherwise.
+     */
+    public static boolean isValid(ByteBuffer term, AbstractType<?> validator)
+    {
+        try
+        {
+            validator.validate(term);
+            return true;
+        }
+        catch (MarshalException e)
+        {
+            return false;
+        }
+    }
+
+    /**
+     * Indicates if the type encoding supports rounding of the raw value.
+     *
+     * This is significant in range searches where we have to make all range
+     * queries inclusive when searching the indexes in order to avoid excluding
+     * rounded values. Excluded values are removed by post-filtering.
+     */
+    public static boolean supportsRounding(AbstractType<?> type)
+    {
+        return isBigInteger(type) || isBigDecimal(type);
+    }
+
+    /**
+     * Returns the smaller of two {@code ByteBuffer} values, based on the result of {@link
+     * #compare(ByteBuffer, ByteBuffer, AbstractType)} comparision.
+     */
+    public static ByteBuffer min(ByteBuffer a, ByteBuffer b, AbstractType<?> type)
+    {
+        return a == null ? b : (b == null || compare(b, a, type) > 0) ? a : b;
+    }
+
+    /**
+     * Returns the greater of two {@code ByteBuffer} values, based on the result of {@link
+     * #compare(ByteBuffer, ByteBuffer, AbstractType)} comparision.
+     */
+    public static ByteBuffer max(ByteBuffer a, ByteBuffer b, AbstractType<?> type)
+    {
+        return a == null ? b : (b == null || compare(b, a, type) < 0) ? a : b;
+    }
+
+    /**
+     * Returns the lesser of two {@code ByteComparable} values, based on the result of {@link
+     * ByteComparable#compare(ByteComparable, ByteComparable, ByteComparable.Version)} comparision.
+     */
+    public static ByteComparable min(ByteComparable a, ByteComparable b)
+    {
+        return a == null ? b : (b == null || ByteComparable.compare(b, a, ByteComparable.Version.OSS42) > 0) ? a : b;
+    }
+
+    /**
+     * Returns the greater of two {@code ByteComparable} values, based on the result of {@link
+     * ByteComparable#compare(ByteComparable, ByteComparable, ByteComparable.Version)} comparision.
+     */
+    public static ByteComparable max(ByteComparable a, ByteComparable b)
+    {
+        return a == null ? b : (b == null || ByteComparable.compare(b, a, ByteComparable.Version.OSS42) < 0) ? a : b;
+    }
+
+    public static AbstractType<?> cellValueType(ColumnMetadata columnMetadata, IndexTarget.Type indexType)
+    {
+        AbstractType<?> type = columnMetadata.type;
+        if (isNonFrozenCollection(type))
+        {
+            CollectionType<?> collection = ((CollectionType<?>) type);
+            switch (collection.kind)
+            {
+                case LIST:
+                    return collection.valueComparator();
+                case SET:
+                    return collection.nameComparator();
+                case MAP:
+                    switch (indexType)
+                    {
+                        case KEYS:
+                            return collection.nameComparator();
+                        case VALUES:
+                            return collection.valueComparator();
+                        case KEYS_AND_VALUES:
+                            return CompositeType.getInstance(collection.nameComparator(), collection.valueComparator());
+                    }
+            }
+        }
+        return type;
+    }
+
+    /**
+     * Allows overriding the default getString method for {@link CompositeType}. It is
+     * a requirement of the {@link ConcurrentRadixTree} that the keys are strings but
+     * the getString method of {@link CompositeType} does not return a string that compares
+     * in the same order as the underlying {@link ByteBuffer}. To get round this we convert
+     * the {@link CompositeType} bytes to a hex string.
+     */
+    public static String getString(ByteBuffer value, AbstractType<?> type)
+    {
+        if (isComposite(type))
+            return ByteBufferUtil.bytesToHex(value);
+        return type.getString(value);
+    }
+
+    /**
+     * The inverse of the above method. Overrides the fromString method on {@link CompositeType}
+     * in order to convert the hex string to bytes.
+     */
+    public static ByteBuffer fromString(String value, AbstractType<?> type)
+    {
+        if (isComposite(type))
+            return ByteBufferUtil.hexToBytes(value);
+        return type.fromString(value);
+    }
+
+    public static ByteSource asComparableBytes(ByteBuffer value, AbstractType<?> type, ByteComparable.Version version)
+    {
+        if (type instanceof InetAddressType || type instanceof IntegerType || type instanceof DecimalType)
+            return ByteSource.optionalFixedLength(ByteBufferAccessor.instance, value);
+        return type.asComparableBytes(value, version);
+    }
+
+    /**
+     * Translates the external value of specific types into a format used by the index.
+     */
+    public static ByteBuffer asIndexBytes(ByteBuffer value, AbstractType<?> type)
+    {
+        if (value == null)
+            return null;
+
+        if (isInetAddress(type))
+            return encodeInetAddress(value);
+        else if (isBigInteger(type))
+            return encodeBigInteger(value);
+        else if (type instanceof DecimalType)
+            return encodeDecimal(value);
+        return value;
+    }
+
+    /**
+     * Compare two terms based on their type. This is used in place of {@link AbstractType#compare(ByteBuffer, ByteBuffer)}
+     * so that the default comparison can be overridden for specific types.
+     *
+     * Note: This should be used for all term comparison
+     */
+    public static int compare(ByteBuffer b1, ByteBuffer b2, AbstractType<?> type)
+    {
+        if (isInetAddress(type))
+            return compareInet(b1, b2);
+        // BigInteger values, frozen types and composite types (map entries) use compareUnsigned to maintain
+        // a consistent order between the in-memory index and the on-disk index.
+        else if (isBigInteger(type) || isBigDecimal(type) || isCompositeOrFrozen(type))
+            return FastByteOperations.compareUnsigned(b1, b2);
+
+        return type.compare(b1, b2 );
+    }
+
+    /**
+     * This is used for value comparison in post-filtering - {@link Expression#isSatisfiedBy(ByteBuffer)}.
+     *
+     * This allows types to decide whether they should be compared based on their encoded value or their
+     * raw value. At present only {@link InetAddressType} values are compared by their encoded values to
+     * allow for ipv4 -> ipv6 equivalency in searches.
+     */
+    public static int comparePostFilter(Expression.Value requestedValue, Expression.Value columnValue, AbstractType<?> type)
+    {
+        if (isInetAddress(type))
+            return compareInet(requestedValue.encoded, columnValue.encoded);
+        // Override comparisons for frozen collections and composite types (map entries)
+        else if (isCompositeOrFrozen(type))
+            return FastByteOperations.compareUnsigned(requestedValue.raw, columnValue.raw);
+
+        return type.compare(requestedValue.raw, columnValue.raw);
+    }
+
+    public static Iterator<ByteBuffer> collectionIterator(AbstractType<?> validator,
+                                                          ComplexColumnData cellData,
+                                                          ColumnMetadata columnMetadata,
+                                                          IndexTarget.Type indexType,
+                                                          int nowInSecs)
+    {
+        if (cellData == null)
+            return null;
+
+        Stream<ByteBuffer> stream = StreamSupport.stream(cellData.spliterator(), false).filter(cell -> cell != null && cell.isLive(nowInSecs))
+                                                 .map(cell -> cellValue(columnMetadata, indexType, cell));
+
+        if (isInetAddress(validator))
+            stream = stream.sorted((c1, c2) -> compareInet(encodeInetAddress(c1), encodeInetAddress(c2)));
+
+        return stream.iterator();
+    }
+
+    public static Comparator<ByteBuffer> comparator(AbstractType<?> type)
+    {
+        // Override the comparator for BigInteger, frozen collections and composite types
+        if (isBigInteger(type) || isBigDecimal(type) || isCompositeOrFrozen(type))
+            return FastByteOperations::compareUnsigned;
+
+        return type;
+    }
+
+    private static ByteBuffer cellValue(ColumnMetadata columnMetadata, IndexTarget.Type indexType, Cell<?> cell)
+    {
+        if (columnMetadata.type.isCollection() && columnMetadata.type.isMultiCell())
+        {
+            switch (((CollectionType<?>) columnMetadata.type).kind)
+            {
+                case LIST:
+                    return cell.buffer();
+                case SET:
+                    return cell.path().get(0);
+                case MAP:
+                    switch (indexType)
+                    {
+                        case KEYS:
+                            return cell.path().get(0);
+                        case VALUES:
+                            return cell.buffer();
+                        case KEYS_AND_VALUES:
+                            return CompositeType.build(ByteBufferAccessor.instance, cell.path().get(0), cell.buffer());
+                    }
+            }
+        }
+        return cell.buffer();
+    }
+
+    /**
+     * Compares 2 InetAddress terms by ensuring that both addresses are represented as
+     * ipv6 addresses.
+     */
+    private static int compareInet(ByteBuffer b1, ByteBuffer b2)
+    {
+        assert isIPv6(b1) && isIPv6(b2);
+
+        return FastByteOperations.compareUnsigned(b1, b2);
+    }
+
+    private static boolean isIPv6(ByteBuffer address)
+    {
+        return address.remaining() == 16;
+    }
+
+    /**
+     * Encode a {@link InetAddress} into a fixed width 16 byte encoded value.
+     *
+     * The encoded value is byte comparable and prefix compressible.
+     *
+     * The encoding is done by converting ipv4 addresses to their ipv6 equivalent.
+     */
+    private static ByteBuffer encodeInetAddress(ByteBuffer value)
+    {
+        if (value.remaining() == 4)
+        {
+            int position = value.hasArray() ? value.arrayOffset() + value.position() : value.position();
+            ByteBuffer mapped = ByteBuffer.allocate(16);
+            System.arraycopy(IPV4_PREFIX, 0, mapped.array(), 0, IPV4_PREFIX.length);
+            ByteBufferUtil.copyBytes(value, position, mapped, IPV4_PREFIX.length, value.remaining());
+            return mapped;
+        }
+        return value;
+    }
+
+    /**
+     * Encode a {@link BigInteger} into a fixed width 20 byte encoded value.
+     *
+     * The encoded value is byte comparable and prefix compressible.
+     *
+     * The format of the encoding is:
+     *
+     *  The first 4 bytes contain the length of the {@link BigInteger} byte array
+     *  with the top bit flipped for positive values.
+     *
+     *  The remaining 16 bytes contain the 16 most significant bytes of the
+     *  {@link BigInteger} byte array.
+     *
+     *  For {@link BigInteger} values whose underlying byte array is less than
+     *  16 bytes, the encoded value is sign extended.
+     */
+    public static ByteBuffer encodeBigInteger(ByteBuffer value)
+    {
+        int size = value.remaining();
+        int position = value.hasArray() ? value.arrayOffset() + value.position() : value.position();
+        byte[] bytes = new byte[20];
+        if (size < 16)
+        {
+            ByteBufferUtil.copyBytes(value, position, bytes, bytes.length - size, size);
+            if ((bytes[bytes.length - size] & 0x80) != 0)
+                Arrays.fill(bytes, 4, bytes.length - size, (byte)0xff);
+            else
+                Arrays.fill(bytes, 4, bytes.length - size, (byte)0x00);
+        }
+        else
+        {
+            ByteBufferUtil.copyBytes(value, position, bytes, 4, 16);
+        }
+        if ((bytes[4] & 0x80) != 0)
+        {
+            size = -size;
+        }
+        bytes[0] = (byte)(size >> 24 & 0xff);
+        bytes[1] = (byte)(size >> 16 & 0xff);
+        bytes[2] = (byte)(size >> 8 & 0xff);
+        bytes[3] = (byte)(size & 0xff);
+        bytes[0] ^= 0x80;
+        return ByteBuffer.wrap(bytes);
+    }
+
+    /**
+     * Returns <code>true</code> if values of the given {@link AbstractType} should be indexed as literals.
+     */
+    public static boolean isLiteral(AbstractType<?> type)
+    {
+        return isUTF8OrAscii(type) || isCompositeOrFrozen(type) || baseType(type) instanceof BooleanType;
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is UTF8 or Ascii
+     */
+    public static boolean isUTF8OrAscii(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type instanceof UTF8Type || type instanceof AsciiType;
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is a Composite(map entry) or frozen.
+     */
+    public static boolean isCompositeOrFrozen(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type instanceof CompositeType || isFrozen(type);
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is frozen.
+     */
+    public static boolean isFrozen(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return !type.subTypes().isEmpty() && !type.isMultiCell();
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is a non-frozen collection.
+     */
+    public static boolean isNonFrozenCollection(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type.isCollection() && type.isMultiCell();
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is {@link InetAddressType}
+     */
+    private static boolean isInetAddress(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type instanceof InetAddressType;
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is {@link IntegerType}
+     */
+    private static boolean isBigInteger(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type instanceof IntegerType;
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is {@link DecimalType}
+     */
+    private static boolean isBigDecimal(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type instanceof DecimalType;
+    }
+
+    /**
+     * Returns <code>true</code> if given {@link AbstractType} is {@link CompositeType}
+     */
+    public static boolean isComposite(AbstractType<?> type)
+    {
+        type = baseType(type);
+        return type instanceof CompositeType;
+    }
+
+    /**
+     * @return base type if given type is reversed, otherwise return itself
+     */
+    private static AbstractType<?> baseType(AbstractType<?> type)
+    {
+        return type.isReversed() ? ((ReversedType<?>) type).baseType : type;
+    }
+
+    public static ByteBuffer encodeDecimal(ByteBuffer value)
+    {
+        ByteSource bs = DecimalType.instance.asComparableBytes(value, ByteComparable.Version.OSS42);
+        bs = ByteSource.cutOrRightPad(bs, DECIMAL_APPROXIMATION_BYTES, 0);
+        return ByteBuffer.wrap(ByteSourceInverse.readBytes(bs, DECIMAL_APPROXIMATION_BYTES));
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/virtual/IndexesSystemView.java b/src/java/org/apache/cassandra/index/sai/virtual/IndexesSystemView.java
new file mode 100644
index 0000000000..eefe888c25
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/virtual/IndexesSystemView.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.virtual;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.marshal.BooleanType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.partitions.PartitionUpdate;
+import org.apache.cassandra.db.virtual.AbstractVirtualTable;
+import org.apache.cassandra.db.virtual.SimpleDataSet;
+import org.apache.cassandra.db.virtual.VirtualTable;
+import org.apache.cassandra.dht.LocalPartitioner;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * A {@link VirtualTable} providing a system view of per-column storage-attached index metadata.
+ */
+public class IndexesSystemView extends AbstractVirtualTable
+{
+    static final String NAME = "indexes";
+
+    static final String KEYSPACE_NAME = "keyspace_name";
+    static final String INDEX_NAME = "index_name";
+    static final String TABLE_NAME = "table_name";
+    static final String COLUMN_NAME = "column_name";
+    static final String IS_QUERYABLE = "is_queryable";
+    static final String IS_BUILDING = "is_building";
+    static final String IS_STRING = "is_string";
+    static final String ANALYZER = "analyzer";
+
+    public IndexesSystemView(String keyspace)
+    {
+        super(TableMetadata.builder(keyspace, NAME)
+                           .partitioner(new LocalPartitioner(UTF8Type.instance))
+                           .comment("Storage-attached column index metadata")
+                           .kind(TableMetadata.Kind.VIRTUAL)
+                           .addPartitionKeyColumn(KEYSPACE_NAME, UTF8Type.instance)
+                           .addClusteringColumn(INDEX_NAME, UTF8Type.instance)
+                           .addRegularColumn(TABLE_NAME, UTF8Type.instance)
+                           .addRegularColumn(COLUMN_NAME, UTF8Type.instance)
+                           .addRegularColumn(IS_QUERYABLE, BooleanType.instance)
+                           .addRegularColumn(IS_BUILDING, BooleanType.instance)
+                           .addRegularColumn(IS_STRING, BooleanType.instance)
+                           .addRegularColumn(ANALYZER, UTF8Type.instance)
+                           .build());
+    }
+
+    @Override
+    public void apply(PartitionUpdate update)
+    {
+        throw new InvalidRequestException("Modification is not supported by table " + metadata);
+    }
+
+    @Override
+    public DataSet data()
+    {
+        SimpleDataSet dataset = new SimpleDataSet(metadata());
+
+        for (KeyspaceMetadata ks : Schema.instance.getUserKeyspaces())
+        {
+            Keyspace keyspace = Schema.instance.getKeyspaceInstance(ks.name);
+            if (keyspace == null)
+                throw new IllegalArgumentException("Unknown keyspace " + ks.name);
+
+            for (ColumnFamilyStore cfs : keyspace.getColumnFamilyStores())
+            {
+                SecondaryIndexManager manager = cfs.indexManager;
+                StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs);
+
+                if (group != null)
+                {
+                    for (Index index : group.getIndexes())
+                    {
+                        IndexContext context = ((StorageAttachedIndex) index).getIndexContext();
+                        String indexName = context.getIndexName();
+
+                        dataset.row(ks.name, indexName)
+                               .column(TABLE_NAME, cfs.name)
+                               .column(COLUMN_NAME, context.getColumnName())
+                               .column(IS_QUERYABLE, manager.isIndexQueryable(index))
+                               .column(IS_BUILDING, manager.isIndexBuilding(indexName))
+                               .column(IS_STRING, context.isLiteral())
+                               .column(ANALYZER, context.getIndexAnalyzerFactory().toString());
+                    }
+                }
+            }
+        }
+
+        return dataset;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/virtual/StorageAttachedIndexTables.java b/src/java/org/apache/cassandra/index/sai/virtual/StorageAttachedIndexTables.java
new file mode 100644
index 0000000000..0d4f12678f
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/virtual/StorageAttachedIndexTables.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.virtual;
+
+import java.util.Collection;
+import java.util.Collections;
+
+import org.apache.cassandra.db.virtual.VirtualTable;
+
+public class StorageAttachedIndexTables
+{
+    private StorageAttachedIndexTables()
+    {}
+
+    public static Collection<VirtualTable> getAll(String keyspace)
+    {
+        return Collections.singletonList(new IndexesSystemView(keyspace));
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java b/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java
index 88b272a769..d76145fd05 100644
--- a/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java
+++ b/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java
@@ -40,15 +40,15 @@ import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.index.sasi.Term;
 import org.apache.cassandra.index.sasi.plan.Expression;
 import org.apache.cassandra.index.sasi.plan.Expression.Op;
-import org.apache.cassandra.index.sasi.utils.AbstractIterator;
 import org.apache.cassandra.index.sasi.utils.MappedBuffer;
-import org.apache.cassandra.index.sasi.utils.RangeIterator;
 import org.apache.cassandra.index.sasi.utils.RangeUnionIterator;
+import org.apache.cassandra.index.sasi.utils.RangeIterator;
 import org.apache.cassandra.io.FSReadError;
 import org.apache.cassandra.io.util.ChannelProxy;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileInputStreamPlus;
 import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.AbstractGuavaIterator;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
 
@@ -731,7 +731,7 @@ public class OnDiskIndex implements Iterable<OnDiskIndex.DataTerm>, Closeable
         return indexPath;
     }
 
-    private class TermIterator extends AbstractIterator<DataTerm>
+    private class TermIterator extends AbstractGuavaIterator<DataTerm>
     {
         private final Expression e;
         private final IteratorOrder order;
diff --git a/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java b/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java
index 3a401cae8a..7f8f3a0f36 100644
--- a/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java
+++ b/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java
@@ -21,10 +21,10 @@ import java.io.IOException;
 import java.util.*;
 
 import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.index.sasi.utils.AbstractIterator;
 import org.apache.cassandra.index.sasi.utils.CombinedValue;
 import org.apache.cassandra.index.sasi.utils.MappedBuffer;
 import org.apache.cassandra.index.sasi.utils.RangeIterator;
+import org.apache.cassandra.utils.AbstractGuavaIterator;
 import org.apache.cassandra.utils.MergeIterator;
 
 import com.carrotsearch.hppc.LongHashSet;
@@ -503,7 +503,7 @@ public class TokenTree
         }
     }
 
-    private static class KeyIterator extends AbstractIterator<DecoratedKey>
+    private static class KeyIterator extends AbstractGuavaIterator<DecoratedKey>
     {
         private final Function<Long, DecoratedKey> keyFetcher;
         private final long[] offsets;
diff --git a/src/java/org/apache/cassandra/index/sasi/memory/KeyRangeIterator.java b/src/java/org/apache/cassandra/index/sasi/memory/KeyRangeIterator.java
index d60914ddcf..b4b1ccb9eb 100644
--- a/src/java/org/apache/cassandra/index/sasi/memory/KeyRangeIterator.java
+++ b/src/java/org/apache/cassandra/index/sasi/memory/KeyRangeIterator.java
@@ -25,12 +25,13 @@ import java.util.concurrent.ConcurrentSkipListSet;
 
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.index.sasi.disk.Token;
-import org.apache.cassandra.index.sasi.utils.AbstractIterator;
 import org.apache.cassandra.index.sasi.utils.CombinedValue;
 import org.apache.cassandra.index.sasi.utils.RangeIterator;
 
 import com.carrotsearch.hppc.LongHashSet;
 import com.carrotsearch.hppc.LongSet;
+import org.apache.cassandra.utils.AbstractGuavaIterator;
+
 import com.google.common.collect.PeekingIterator;
 
 public class KeyRangeIterator extends RangeIterator<Long, Token>
@@ -64,7 +65,7 @@ public class KeyRangeIterator extends RangeIterator<Long, Token>
     public void close() throws IOException
     {}
 
-    private static class DKIterator extends AbstractIterator<DecoratedKey> implements PeekingIterator<DecoratedKey>
+    private static class DKIterator extends AbstractGuavaIterator<DecoratedKey> implements PeekingIterator<DecoratedKey>
     {
         private final Iterator<DecoratedKey> keys;
 
diff --git a/src/java/org/apache/cassandra/index/sasi/utils/RangeIterator.java b/src/java/org/apache/cassandra/index/sasi/utils/RangeIterator.java
index 06c5a6a343..c7db773705 100644
--- a/src/java/org/apache/cassandra/index/sasi/utils/RangeIterator.java
+++ b/src/java/org/apache/cassandra/index/sasi/utils/RangeIterator.java
@@ -24,7 +24,9 @@ import java.util.PriorityQueue;
 
 import com.google.common.annotations.VisibleForTesting;
 
-public abstract class RangeIterator<K extends Comparable<K>, T extends CombinedValue<K>> extends AbstractIterator<T> implements Closeable
+import org.apache.cassandra.utils.AbstractGuavaIterator;
+
+public abstract class RangeIterator<K extends Comparable<K>, T extends CombinedValue<K>> extends AbstractGuavaIterator<T> implements Closeable
 {
     private final K min, max;
     private final long count;
diff --git a/src/java/org/apache/cassandra/locator/ReplicaPlans.java b/src/java/org/apache/cassandra/locator/ReplicaPlans.java
index c39862adff..594c49d6c9 100644
--- a/src/java/org/apache/cassandra/locator/ReplicaPlans.java
+++ b/src/java/org/apache/cassandra/locator/ReplicaPlans.java
@@ -37,6 +37,8 @@ import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.exceptions.UnavailableException;
 import org.apache.cassandra.gms.FailureDetector;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.IndexStatusManager;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.service.reads.AlwaysSpeculativeRetryPolicy;
@@ -56,6 +58,8 @@ import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Predicate;
 
+import javax.annotation.Nullable;
+
 import static com.google.common.collect.Iterables.any;
 import static com.google.common.collect.Iterables.filter;
 import static org.apache.cassandra.db.ConsistencyLevel.EACH_QUORUM;
@@ -522,12 +526,14 @@ public class ReplicaPlans
         return new ReplicaPlan.ForPaxosWrite(keyspace, consistencyForPaxos, liveAndDown.pending(), liveAndDown.all(), live.all(), contacts, requiredParticipants);
     }
 
-
-    private static <E extends Endpoints<E>> E candidatesForRead(ConsistencyLevel consistencyLevel, E liveNaturalReplicas)
+    private static <E extends Endpoints<E>> E candidatesForRead(Keyspace keyspace,
+                                                                @Nullable Index.QueryPlan indexQueryPlan,
+                                                                ConsistencyLevel consistencyLevel,
+                                                                E liveNaturalReplicas)
     {
-        return consistencyLevel.isDatacenterLocal()
-                ? liveNaturalReplicas.filter(InOurDc.replicas())
-                : liveNaturalReplicas;
+        E replicas = consistencyLevel.isDatacenterLocal() ? liveNaturalReplicas.filter(InOurDc.replicas()) : liveNaturalReplicas;
+
+        return indexQueryPlan != null ? IndexStatusManager.instance.filterForQuery(replicas, keyspace, indexQueryPlan, consistencyLevel) : replicas;
     }
 
     private static <E extends Endpoints<E>> E contactForEachQuorumRead(NetworkTopologyStrategy replicationStrategy, E candidates)
@@ -587,10 +593,14 @@ public class ReplicaPlans
      * The candidate collection can be used for speculation, although at present
      * it would break EACH_QUORUM to do so without further filtering
      */
-    public static ReplicaPlan.ForTokenRead forRead(Keyspace keyspace, Token token, ConsistencyLevel consistencyLevel, SpeculativeRetryPolicy retry)
+    public static ReplicaPlan.ForTokenRead forRead(Keyspace keyspace,
+                                                   Token token,
+                                                   @Nullable Index.QueryPlan indexQueryPlan,
+                                                   ConsistencyLevel consistencyLevel,
+                                                   SpeculativeRetryPolicy retry)
     {
         AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy();
-        EndpointsForToken candidates = candidatesForRead(consistencyLevel, ReplicaLayout.forTokenReadLiveSorted(replicationStrategy, token).natural());
+        EndpointsForToken candidates = candidatesForRead(keyspace, indexQueryPlan, consistencyLevel, ReplicaLayout.forTokenReadLiveSorted(replicationStrategy, token).natural());
         EndpointsForToken contacts = contactForRead(replicationStrategy, consistencyLevel, retry.equals(AlwaysSpeculativeRetryPolicy.INSTANCE), candidates);
 
         assureSufficientLiveReplicasForRead(replicationStrategy, consistencyLevel, contacts);
@@ -604,10 +614,14 @@ public class ReplicaPlans
      *
      * There is no speculation for range read queries at present, so we never 'always speculate' here, and a failed response fails the query.
      */
-    public static ReplicaPlan.ForRangeRead forRangeRead(Keyspace keyspace, ConsistencyLevel consistencyLevel, AbstractBounds<PartitionPosition> range, int vnodeCount)
+    public static ReplicaPlan.ForRangeRead forRangeRead(Keyspace keyspace,
+                                                        @Nullable Index.QueryPlan indexQueryPlan,
+                                                        ConsistencyLevel consistencyLevel,
+                                                        AbstractBounds<PartitionPosition> range,
+                                                        int vnodeCount)
     {
         AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy();
-        EndpointsForRange candidates = candidatesForRead(consistencyLevel, ReplicaLayout.forRangeReadLiveSorted(replicationStrategy, range).natural());
+        EndpointsForRange candidates = candidatesForRead(keyspace, indexQueryPlan, consistencyLevel, ReplicaLayout.forRangeReadLiveSorted(replicationStrategy, range).natural());
         EndpointsForRange contacts = contactForRead(replicationStrategy, consistencyLevel, false, candidates);
 
         assureSufficientLiveReplicasForRead(replicationStrategy, consistencyLevel, contacts);
diff --git a/src/java/org/apache/cassandra/schema/IndexMetadata.java b/src/java/org/apache/cassandra/schema/IndexMetadata.java
index cdba139b5d..577c12b4ba 100644
--- a/src/java/org/apache/cassandra/schema/IndexMetadata.java
+++ b/src/java/org/apache/cassandra/schema/IndexMetadata.java
@@ -36,8 +36,11 @@ import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.cql3.CqlBuilder;
 import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.exceptions.UnknownIndexException;
 import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.internal.CassandraIndex;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
 import org.apache.cassandra.index.sasi.SASIIndex;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
@@ -64,6 +67,7 @@ public final class IndexMetadata
 
     static
     {
+        indexNameAliases.put(StorageAttachedIndex.class.getSimpleName(), StorageAttachedIndex.class.getCanonicalName());
         indexNameAliases.put(SASIIndex.class.getSimpleName(), SASIIndex.class.getCanonicalName());
     }
 
@@ -135,8 +139,8 @@ public final class IndexMetadata
                 throw new ConfigurationException(String.format("Required option missing for index %s : %s",
                                                                name, IndexTarget.CUSTOM_INDEX_OPTION_NAME));
 
-            // Find any aliases to the fully qualified index class name:
-            String className = expandAliases(options.get(IndexTarget.CUSTOM_INDEX_OPTION_NAME));
+            // Get the fully qualified class name:
+            String className = getIndexClassName();
 
             Class<Index> indexerClass = FBUtilities.classForName(className, "custom indexer");
             if (!Index.class.isAssignableFrom(indexerClass))
@@ -145,9 +149,14 @@ public final class IndexMetadata
         }
     }
 
-    public static String expandAliases(String className)
+    public String getIndexClassName()
     {
-        return indexNameAliases.getOrDefault(className, className);
+        if (isCustom())
+        {
+            String className = options.get(IndexTarget.CUSTOM_INDEX_OPTION_NAME);
+            return indexNameAliases.getOrDefault(className, className);
+        }
+        return CassandraIndex.class.getName();
     }
 
     private void validateCustomIndexOptions(TableMetadata table, Class<? extends Index> indexerClass, Map<String, String> options)
@@ -179,6 +188,8 @@ public final class IndexMetadata
         }
         catch (InvocationTargetException e)
         {
+            if (e.getTargetException() instanceof InvalidRequestException)
+                throw (InvalidRequestException) e.getTargetException();
             if (e.getTargetException() instanceof ConfigurationException)
                 throw (ConfigurationException) e.getTargetException();
             throw new ConfigurationException("Failed to validate custom indexer options: " + options);
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index 5f99fef4bc..ed73467a72 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -146,6 +146,7 @@ import org.apache.cassandra.gms.IFailureDetector;
 import org.apache.cassandra.gms.TokenSerializer;
 import org.apache.cassandra.gms.VersionedValue;
 import org.apache.cassandra.hints.HintsService;
+import org.apache.cassandra.index.IndexStatusManager;
 import org.apache.cassandra.io.sstable.IScrubber;
 import org.apache.cassandra.io.sstable.IVerifier;
 import org.apache.cassandra.io.sstable.SSTableLoader;
@@ -626,7 +627,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
      * they get the Gossip shutdown message, so even if
      * we don't get time to broadcast this, it is not a problem.
      *
-     * See {@link Gossiper#markAsShutdown(InetAddressAndPort)}
+     * See {@code Gossiper#markAsShutdown(InetAddressAndPort)}
      */
     private void shutdownClientServers()
     {
@@ -2706,6 +2707,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         }
         else
         {
+            if (state == ApplicationState.INDEX_STATUS)
+            {
+                updateIndexStatus(endpoint, value);
+                return;
+            }
+
             EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(endpoint);
             if (epState == null || Gossiper.instance.isDeadState(epState))
             {
@@ -2776,6 +2783,11 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         return value.value.split(VersionedValue.DELIMITER_STR, -1);
     }
 
+    private void updateIndexStatus(InetAddressAndPort endpoint, VersionedValue versionedValue)
+    {
+        IndexStatusManager.instance.receivePeerIndexStatus(endpoint, versionedValue);
+    }
+
     private void updateNetVersion(InetAddressAndPort endpoint, VersionedValue value)
     {
         try
@@ -2848,6 +2860,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
                 case HOST_ID:
                     SystemKeyspace.updatePeerInfo(endpoint, "host_id", UUID.fromString(entry.getValue().value));
                     break;
+                case INDEX_STATUS:
+                    // Need to set the peer index status in SIM here
+                    // to ensure the status is correct before the node
+                    // fully joins the ring
+                    updateIndexStatus(endpoint, entry.getValue());
+                    break;
             }
         }
 
diff --git a/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java b/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java
index 1613283300..8fc5d1d89b 100644
--- a/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java
+++ b/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java
@@ -181,13 +181,19 @@ public abstract class AbstractReadExecutor
     /**
      * @return an executor appropriate for the configured speculative read policy
      */
-    public static AbstractReadExecutor getReadExecutor(SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel, long queryStartNanoTime) throws UnavailableException
+    public static AbstractReadExecutor getReadExecutor(SinglePartitionReadCommand command,
+                                                       ConsistencyLevel consistencyLevel,
+                                                       long queryStartNanoTime) throws UnavailableException
     {
         Keyspace keyspace = Keyspace.open(command.metadata().keyspace);
         ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.metadata().id);
         SpeculativeRetryPolicy retry = cfs.metadata().params.speculativeRetry;
 
-        ReplicaPlan.ForTokenRead replicaPlan = ReplicaPlans.forRead(keyspace, command.partitionKey().getToken(), consistencyLevel, retry);
+        ReplicaPlan.ForTokenRead replicaPlan = ReplicaPlans.forRead(keyspace,
+                                                                    command.partitionKey().getToken(),
+                                                                    command.indexQueryPlan(),
+                                                                    consistencyLevel,
+                                                                    retry);
 
         // Speculative retry is disabled *OR*
         // 11980: Disable speculative retry if using EACH_QUORUM in order to prevent miscounting DC responses
diff --git a/src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java b/src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java
index 889fa79a50..67815ac925 100644
--- a/src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java
+++ b/src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java
@@ -523,7 +523,7 @@ public class ReplicaFilteringProtection<E extends Endpoints<E>>
             SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(command.metadata(),
                                                                                command.nowInSec(),
                                                                                command.columnFilter(),
-                                                                               RowFilter.NONE,
+                                                                               RowFilter.none(),
                                                                                limits,
                                                                                key,
                                                                                filter);
diff --git a/src/java/org/apache/cassandra/service/reads/range/RangeCommands.java b/src/java/org/apache/cassandra/service/reads/range/RangeCommands.java
index ca3c220d70..865b03d487 100644
--- a/src/java/org/apache/cassandra/service/reads/range/RangeCommands.java
+++ b/src/java/org/apache/cassandra/service/reads/range/RangeCommands.java
@@ -74,7 +74,10 @@ public class RangeCommands
         Tracing.trace("Computing ranges to query");
 
         Keyspace keyspace = Keyspace.open(command.metadata().keyspace);
-        ReplicaPlanIterator replicaPlans = new ReplicaPlanIterator(command.dataRange().keyRange(), keyspace, consistencyLevel);
+        ReplicaPlanIterator replicaPlans = new ReplicaPlanIterator(command.dataRange().keyRange(),
+                                                                   command.indexQueryPlan(),
+                                                                   keyspace,
+                                                                   consistencyLevel);
 
         // our estimate of how many result rows there will be per-range
         float resultsPerRange = estimateResultsPerRange(command, keyspace);
@@ -128,11 +131,13 @@ public class RangeCommands
         {
             Keyspace keyspace = Keyspace.open(metadata.keyspace);
             ReplicaPlanIterator rangeIterator = new ReplicaPlanIterator(DataRange.allData(metadata.partitioner).keyRange(),
-                                                                        keyspace, consistency);
+                                                                        null,
+                                                                        keyspace,
+                                                                        consistency);
 
             // Called for the side effect of running assureSufficientLiveReplicasForRead.
             // Deliberately called with an invalid vnode count in case it is used elsewhere in the future..
-            rangeIterator.forEachRemaining(r ->  ReplicaPlans.forRangeRead(keyspace, consistency, r.range(), -1));
+            rangeIterator.forEachRemaining(r ->  ReplicaPlans.forRangeRead(keyspace, null, consistency, r.range(), -1));
             return true;
         }
         catch (UnavailableException e)
diff --git a/src/java/org/apache/cassandra/service/reads/range/ReplicaPlanIterator.java b/src/java/org/apache/cassandra/service/reads/range/ReplicaPlanIterator.java
index ef88c9dffd..c14dc3c4c8 100644
--- a/src/java/org/apache/cassandra/service/reads/range/ReplicaPlanIterator.java
+++ b/src/java/org/apache/cassandra/service/reads/range/ReplicaPlanIterator.java
@@ -23,6 +23,8 @@ import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
+import javax.annotation.Nullable;
+
 import com.google.common.annotations.VisibleForTesting;
 
 import org.apache.cassandra.db.ConsistencyLevel;
@@ -31,6 +33,7 @@ import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.dht.Bounds;
 import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.locator.LocalStrategy;
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.locator.ReplicaPlans;
@@ -43,12 +46,17 @@ class ReplicaPlanIterator extends AbstractIterator<ReplicaPlan.ForRangeRead>
 {
     private final Keyspace keyspace;
     private final ConsistencyLevel consistency;
+    private final Index.QueryPlan indexQueryPlan;
     @VisibleForTesting
     final Iterator<? extends AbstractBounds<PartitionPosition>> ranges;
     private final int rangeCount;
 
-    ReplicaPlanIterator(AbstractBounds<PartitionPosition> keyRange, Keyspace keyspace, ConsistencyLevel consistency)
+    ReplicaPlanIterator(AbstractBounds<PartitionPosition> keyRange,
+                        @Nullable Index.QueryPlan indexQueryPlan,
+                        Keyspace keyspace,
+                        ConsistencyLevel consistency)
     {
+        this.indexQueryPlan = indexQueryPlan;
         this.keyspace = keyspace;
         this.consistency = consistency;
 
@@ -73,7 +81,7 @@ class ReplicaPlanIterator extends AbstractIterator<ReplicaPlan.ForRangeRead>
         if (!ranges.hasNext())
             return endOfData();
 
-        return ReplicaPlans.forRangeRead(keyspace, consistency, ranges.next(), 1);
+        return ReplicaPlans.forRangeRead(keyspace, indexQueryPlan, consistency, ranges.next(), 1);
     }
 
     /**
diff --git a/src/java/org/apache/cassandra/index/sasi/utils/AbstractIterator.java b/src/java/org/apache/cassandra/utils/AbstractGuavaIterator.java
similarity index 83%
rename from src/java/org/apache/cassandra/index/sasi/utils/AbstractIterator.java
rename to src/java/org/apache/cassandra/utils/AbstractGuavaIterator.java
index cf918c133b..4651e14f71 100644
--- a/src/java/org/apache/cassandra/index/sasi/utils/AbstractIterator.java
+++ b/src/java/org/apache/cassandra/utils/AbstractGuavaIterator.java
@@ -1,11 +1,15 @@
 /*
  * Copyright (C) 2007 The Guava Authors
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
  *
- * http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,7 +18,7 @@
  * limitations under the License.
  */
 
-package org.apache.cassandra.index.sasi.utils;
+package org.apache.cassandra.utils;
 
 import java.util.NoSuchElementException;
 
@@ -22,15 +26,17 @@ import com.google.common.collect.PeekingIterator;
 
 import static com.google.common.base.Preconditions.checkState;
 
-// This is fork of the Guava AbstractIterator, the only difference
-// is that state & next variables are now protected, this was required
-// for SkippableIterator.skipTo(..) to void all previous state.
-public abstract class AbstractIterator<T> implements PeekingIterator<T>
+/**
+ * This is fork of the Guava AbstractIterator, the only difference
+ * is that the next variable is now protected so that the KeyRangeIterator.skipTo
+ * method can avoid early state changed.
+ */
+public abstract class AbstractGuavaIterator<T> implements PeekingIterator<T>
 {
-    protected State state = State.NOT_READY;
+    private State state = State.NOT_READY;
 
     /** Constructor for use by subclasses. */
-    protected AbstractIterator() {}
+    protected AbstractGuavaIterator() {}
 
     protected enum State
     {
diff --git a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
index 790ff2ae02..31c341eb51 100644
--- a/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
+++ b/src/java/org/apache/cassandra/utils/ByteBufferUtil.java
@@ -280,6 +280,15 @@ public class ByteBufferUtil
         return clone;
     }
 
+    /**
+     * Transfer bytes from a ByteBuffer to byte array.
+     *
+     * @param src the source ByteBuffer
+     * @param srcPos starting position in the source ByteBuffer
+     * @param dst the destination byte array
+     * @param dstPos starting position in the destination byte array
+     * @param length the number of bytes to copy
+     */
     public static void copyBytes(ByteBuffer src, int srcPos, byte[] dst, int dstPos, int length)
     {
         FastByteOperations.copy(src, srcPos, dst, dstPos, length);
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/IndexAvailabilityTest.java b/test/distributed/org/apache/cassandra/distributed/test/sai/IndexAvailabilityTest.java
new file mode 100644
index 0000000000..3a9e111bad
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/IndexAvailabilityTest.java
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.sai;
+
+import java.net.InetAddress;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.Objects;
+import org.junit.Test;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.IndexStatusManager;
+import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.FBUtilities;
+
+import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
+import static org.apache.cassandra.distributed.api.Feature.NETWORK;
+import static org.apache.cassandra.distributed.test.sai.SAIUtil.waitForIndexQueryable;
+import static org.awaitility.Awaitility.await;
+import static org.junit.Assert.assertEquals;
+
+public class IndexAvailabilityTest extends TestBaseImpl
+{
+    private static final String CREATE_KEYSPACE = "CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': %d}";
+    private static final String CREATE_TABLE = "CREATE TABLE %s.%s (pk text primary key, v1 int, v2 text) " +
+                                               "WITH compaction = {'class' : 'SizeTieredCompactionStrategy', 'enabled' : false }";
+    private static final String CREATE_INDEX = "CREATE CUSTOM INDEX %s ON %s.%s(%s) USING 'StorageAttachedIndex'";
+    
+    private static final Map<NodeIndex, Index.Status> expectedNodeIndexQueryability = new ConcurrentHashMap<>();
+    private List<String> keyspaces;
+    private List<String> indexesPerKs;
+
+    @Test
+    public void verifyIndexStatusPropagation() throws Exception
+    {
+        try (Cluster cluster = init(Cluster.build(2)
+                                           .withConfig(config -> config.with(GOSSIP)
+                                                                       .with(NETWORK))
+                                           .start()))
+        {
+            String ks1 = "ks1";
+            String ks2 = "ks2";
+            String ks3 = "ks3";
+            String cf1 = "cf1";
+            String index1 = "cf1_idx1";
+            String index2 = "cf1_idx2";
+
+            keyspaces = Arrays.asList(ks1, ks2, ks3);
+            indexesPerKs = Arrays.asList(index1, index2);
+
+            // create 1 tables per keyspace, 2 indexes per table. all indexes are queryable
+            for (String ks : keyspaces)
+            {
+                cluster.schemaChange(String.format(CREATE_KEYSPACE, ks, 2));
+                cluster.schemaChange(String.format(CREATE_TABLE, ks, cf1));
+                cluster.schemaChange(String.format(CREATE_INDEX, index1, ks, cf1, "v1"));
+                cluster.schemaChange(String.format(CREATE_INDEX, index2, ks, cf1, "v2"));
+                waitForIndexQueryable(cluster, ks);
+                cluster.forEach(node -> {
+                    expectedNodeIndexQueryability.put(NodeIndex.create(ks, index1, node), Index.Status.BUILD_SUCCEEDED);
+                    expectedNodeIndexQueryability.put(NodeIndex.create(ks, index2, node), Index.Status.BUILD_SUCCEEDED);
+                });
+            }
+
+            // mark ks1 index1 as non-queryable on node1
+            markIndexNonQueryable(cluster.get(1), ks1, cf1, index1);
+            // on node2, it observes that node1 ks1.index1 is not queryable
+            waitForIndexingStatus(cluster.get(2), ks1, index1, cluster.get(1), Index.Status.BUILD_FAILED);
+            // other indexes or keyspaces should not be affected
+            assertIndexingStatus(cluster);
+
+            // mark ks2 index2 as non-queryable on node2
+            markIndexNonQueryable(cluster.get(2), ks2, cf1, index2);
+            // on node1, it observes that node2 ks2.index2 is not queryable
+            waitForIndexingStatus(cluster.get(1), ks2, index2, cluster.get(2), Index.Status.BUILD_FAILED);
+            // other indexes or keyspaces should not be affected
+            assertIndexingStatus(cluster);
+
+            // mark ks1 index1 as queryable on node1
+            markIndexQueryable(cluster.get(1), ks1, cf1, index1);
+            // on node2, it observes that node1 ks1.index1 is queryable
+            waitForIndexingStatus(cluster.get(2), ks1, index1, cluster.get(1), Index.Status.BUILD_SUCCEEDED);
+            // other indexes or keyspaces should not be affected
+            assertIndexingStatus(cluster);
+
+            // mark ks2 index2 as indexing on node1
+            markIndexBuilding(cluster.get(1), ks2, cf1, index2);
+            // on node2, it observes that node1 ks2.index2 is not queryable
+            waitForIndexingStatus(cluster.get(2), ks2, index2, cluster.get(1), Index.Status.FULL_REBUILD_STARTED);
+            // other indexes or keyspaces should not be affected
+            assertIndexingStatus(cluster);
+
+            // drop ks1, ks1 index1/index2 should be non queryable on all nodes
+            cluster.schemaChange("DROP KEYSPACE " + ks1);
+            expectedNodeIndexQueryability.keySet().forEach(k -> {
+                if (k.keyspace.equals(ks1))
+                    expectedNodeIndexQueryability.put(k, Index.Status.UNKNOWN);
+            });
+            assertIndexingStatus(cluster);
+
+            // drop ks2 index2, there should be no ks2 index2 status on all node
+            cluster.schemaChange("DROP INDEX " + ks2 + "." + index2);
+            expectedNodeIndexQueryability.keySet().forEach(k -> {
+                if (k.keyspace.equals(ks2) && k.index.equals(index2))
+                    expectedNodeIndexQueryability.put(k, Index.Status.UNKNOWN);
+            });
+            assertIndexingStatus(cluster);
+
+            // drop ks3 cf1, there should be no ks3 index1/index2 status
+            cluster.schemaChange("DROP TABLE " + ks3 + "." + cf1);
+            expectedNodeIndexQueryability.keySet().forEach(k -> {
+                if (k.keyspace.equals(ks3))
+                    expectedNodeIndexQueryability.put(k, Index.Status.UNKNOWN);
+            });
+            assertIndexingStatus(cluster);
+        }
+    }
+
+    private void markIndexNonQueryable(IInvokableInstance node, String keyspace, String table, String indexName)
+    {
+        expectedNodeIndexQueryability.put(NodeIndex.create(keyspace, indexName, node), Index.Status.BUILD_FAILED);
+
+        node.runOnInstance(() -> {
+            SecondaryIndexManager sim = Schema.instance.getKeyspaceInstance(keyspace).getColumnFamilyStore(table).indexManager;
+            Index index = sim.getIndexByName(indexName);
+            sim.makeIndexNonQueryable(index, Index.Status.BUILD_FAILED);
+        });
+    }
+
+    private void markIndexQueryable(IInvokableInstance node, String keyspace, String table, String indexName)
+    {
+        expectedNodeIndexQueryability.put(NodeIndex.create(keyspace, indexName, node), Index.Status.BUILD_SUCCEEDED);
+
+        node.runOnInstance(() -> {
+            SecondaryIndexManager sim = Schema.instance.getKeyspaceInstance(keyspace).getColumnFamilyStore(table).indexManager;
+            Index index = sim.getIndexByName(indexName);
+            sim.makeIndexQueryable(index, Index.Status.BUILD_SUCCEEDED);
+        });
+    }
+
+    private void markIndexBuilding(IInvokableInstance node, String keyspace, String table, String indexName)
+    {
+        expectedNodeIndexQueryability.put(NodeIndex.create(keyspace, indexName, node), Index.Status.FULL_REBUILD_STARTED);
+
+        node.runOnInstance(() -> {
+            SecondaryIndexManager sim = Schema.instance.getKeyspaceInstance(keyspace).getColumnFamilyStore(table).indexManager;
+            Index index = sim.getIndexByName(indexName);
+            sim.markIndexesBuilding(Collections.singleton(index), true, false);
+        });
+    }
+
+    private void assertIndexingStatus(Cluster cluster)
+    {
+        for (String ks : keyspaces)
+        {
+            for (String indexName : indexesPerKs)
+            {
+                assertIndexingStatus(cluster, ks, indexName);
+            }
+        }
+    }
+
+    private static void assertIndexingStatus(Cluster cluster, String keyspace, String indexName)
+    {
+        for (int nodeId = 1; nodeId <= cluster.size(); nodeId++)
+        {
+            for (int replica = 1; replica <= cluster.size(); replica++)
+            {
+                NodeIndex nodeIndex = NodeIndex.create(keyspace, indexName, cluster.get(replica));
+                Index.Status expected = expectedNodeIndexQueryability.get(nodeIndex);
+
+                assertIndexingStatus(cluster.get(nodeId), keyspace, indexName, cluster.get(replica), expected);
+            }
+        }
+    }
+
+    private static void assertIndexingStatus(IInvokableInstance node, String keyspaceName, String indexName, IInvokableInstance replica, Index.Status expected)
+    {
+        InetAddressAndPort replicaAddressAndPort = getFullAddress(replica);
+        try
+        {
+            Index.Status actual = getNodeIndexStatus(node, keyspaceName, indexName, replicaAddressAndPort);
+            String errorMessage = String.format("Failed to verify %s.%s status for replica %s on node %s, expected %s, but got %s.",
+                                                keyspaceName, indexName, replica.broadcastAddress(), node.broadcastAddress(), expected, actual);
+            assertEquals(errorMessage, expected, actual);
+        }
+        catch (Exception e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static void waitForIndexingStatus(IInvokableInstance node, String keyspace, String index, IInvokableInstance replica, Index.Status status)
+    {
+        InetAddressAndPort replicaAddressAndPort = getFullAddress(replica);
+        await().atMost(5, TimeUnit.SECONDS)
+               .until(() -> node.callOnInstance(() -> getIndexStatus(keyspace, index, replicaAddressAndPort) == status));
+    }
+
+    private static Index.Status getNodeIndexStatus(IInvokableInstance node, String keyspaceName, String indexName, InetAddressAndPort replica)
+    {
+        return Index.Status.values()[node.callsOnInstance(() -> getIndexStatus(keyspaceName, indexName, replica).ordinal()).call()];
+    }
+    
+    private static Index.Status getIndexStatus(String keyspaceName, String indexName, InetAddressAndPort replica)
+    {
+        KeyspaceMetadata keyspace = Schema.instance.getKeyspaceMetadata(keyspaceName);
+        if (keyspace == null)
+            return Index.Status.UNKNOWN;
+
+        TableMetadata table = keyspace.findIndexedTable(indexName).orElse(null);
+        if (table == null)
+            return Index.Status.UNKNOWN;
+
+        return IndexStatusManager.instance.getIndexStatus(replica, keyspaceName, indexName);
+    }
+
+    private static InetAddressAndPort getFullAddress(IInvokableInstance node)
+    {
+        InetAddress address = node.broadcastAddress().getAddress();
+        int port = node.callOnInstance(() -> FBUtilities.getBroadcastAddressAndPort().getPort());
+        return InetAddressAndPort.getByAddressOverrideDefaults(address, port);
+    }
+    
+    private static class NodeIndex
+    {
+        private final String keyspace;
+        private final String index;
+        private final IInvokableInstance node;
+
+        NodeIndex(String keyspace, String index, IInvokableInstance node)
+        {
+            this.keyspace = keyspace;
+            this.index = index;
+            this.node = node;
+        }
+
+        public static NodeIndex create(String keyspace, String index, IInvokableInstance node)
+        {
+            return new NodeIndex(keyspace, index, node);
+        }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            NodeIndex that = (NodeIndex) o;
+            return node.equals(that.node) &&
+                   Objects.equal(keyspace, that.keyspace) &&
+                   Objects.equal(index, that.index);
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hashCode(keyspace, index, node);
+        }
+    }
+}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/sai/SAIUtil.java b/test/distributed/org/apache/cassandra/distributed/test/sai/SAIUtil.java
new file mode 100644
index 0000000000..c6ce764c75
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/test/sai/SAIUtil.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.test.sai;
+
+import java.net.InetSocketAddress;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.api.IInstance;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.distributed.api.SimpleQueryResult;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.IndexStatusManager;
+import org.apache.cassandra.index.SecondaryIndexManager;
+import org.apache.cassandra.locator.InetAddressAndPort;
+import org.assertj.core.util.Streams;
+
+import static org.awaitility.Awaitility.await;
+
+public class SAIUtil
+{
+    /**
+     * Waits until all indexes in the given keyspace become queryable.
+     */
+    public static void waitForIndexQueryable(Cluster cluster, String keyspace)
+    {
+        assertGossipEnabled(cluster);
+        final List<String> indexes = getIndexes(cluster, keyspace);
+        await().atMost(60, TimeUnit.SECONDS)
+               .untilAsserted(() -> assertIndexesQueryable(cluster, keyspace, indexes));
+    }
+
+    /**
+     * Waits until given index becomes queryable.
+     */
+    public static void waitForIndexQueryable(Cluster cluster, String keyspace, String index)
+    {
+        assertGossipEnabled(cluster);
+        await().atMost(60, TimeUnit.SECONDS)
+               .untilAsserted(() -> assertIndexQueryable(cluster, keyspace, index));
+    }
+
+    private static void assertGossipEnabled(Cluster cluster)
+    {
+        cluster.stream().forEach(node -> {
+            assert node.config().has(Feature.NETWORK) : "Network not enabled on this cluster";
+            assert node.config().has(Feature.GOSSIP) : "Gossip not enabled on this cluster";
+        });
+    }
+
+    /**
+     * Checks if index is known to be queryable, by pulling index state from {{@link SecondaryIndexManager}}.
+     * Requires gossip.
+     */
+    public static void assertIndexQueryable(Cluster cluster, String keyspace, String index)
+    {
+        assertIndexesQueryable(cluster, keyspace, Collections.singleton(index));
+    }
+
+    /**
+     * Checks if all indexes are known to be queryable, by pulling index state from local {{@link SecondaryIndexManager}}.
+     * Requires gossip.
+     */
+    private static void assertIndexesQueryable(Cluster cluster, String keyspace, final Iterable<String> indexes)
+    {
+        IInvokableInstance localNode = cluster.get(1);
+        final List<InetAddressAndPort> nodes =
+            cluster.stream()
+                   .map(node -> nodeAddress(node.broadcastAddress()))
+                   .collect(Collectors.toList());
+
+        localNode.runOnInstance(() -> {
+            for (String index : indexes)
+            {
+                for (InetAddressAndPort node : nodes)
+                {
+                    Index.Status status = IndexStatusManager.instance.getIndexStatus(node, keyspace, index);
+                    assert status == Index.Status.BUILD_SUCCEEDED
+                        : "Index " + index + " not queryable on node " + node + " (status = " + status + ')';
+                }
+            }
+        });
+    }
+
+    private static InetAddressAndPort nodeAddress(InetSocketAddress address)
+    {
+        return InetAddressAndPort.getByAddressOverrideDefaults(address.getAddress(), address.getPort());
+    }
+
+    /**
+     * Returns names of the indexes in the keyspace, found on the first node of the cluster.
+     */
+    public static List<String> getIndexes(Cluster cluster, String keyspace)
+    {
+        waitForSchemaAgreement(cluster);
+        String query = String.format("SELECT index_name FROM system_views.indexes WHERE keyspace_name = '%s' ALLOW FILTERING", keyspace);
+        SimpleQueryResult result = cluster.get(1).executeInternalWithResult(query);
+        return Streams.stream(result)
+                      .map(row -> (String) row.get("index_name"))
+                      .collect(Collectors.toList());
+    }
+
+    public static void waitForSchemaAgreement(Cluster cluster)
+    {
+        await().atMost(60, TimeUnit.SECONDS)
+               .until(() -> schemaAgrees(cluster));
+    }
+
+    /**
+     * Returns true if schema agrees on all nodes of the cluster
+     */
+    public static boolean schemaAgrees(Cluster cluster)
+    {
+        Set<UUID> versions = cluster.stream()
+                                    .map(IInstance::schemaVersion)
+                                    .collect(Collectors.toSet());
+        return versions.size() == 1;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java
index 41b86f1c6e..a898faeb36 100644
--- a/test/unit/org/apache/cassandra/cql3/CQLTester.java
+++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java
@@ -2417,6 +2417,21 @@ public abstract class CQLTester
         {
             return "TupleValue" + toCQLString();
         }
+
+        @Override
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            TupleValue that = (TupleValue) o;
+            return Arrays.equals(values, that.values);
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hashCode(values);
+        }
     }
 
     private static class UserTypeValue extends TupleValue
diff --git a/test/unit/org/apache/cassandra/db/KeyspaceTest.java b/test/unit/org/apache/cassandra/db/KeyspaceTest.java
index 2c5e79310d..55baba8469 100644
--- a/test/unit/org/apache/cassandra/db/KeyspaceTest.java
+++ b/test/unit/org/apache/cassandra/db/KeyspaceTest.java
@@ -297,7 +297,7 @@ public class KeyspaceTest extends CQLTester
                          ? DataLimits.NONE
                          : DataLimits.cqlLimits(rowLimit);
         return SinglePartitionReadCommand.create(
-                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, limit, Util.dk(key), filter);
+                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.none(), limit, Util.dk(key), filter);
     }
 
     @Test
@@ -483,17 +483,17 @@ public class KeyspaceTest extends CQLTester
     {
         ClusteringIndexSliceFilter filter = slices(cfs, 1000, null, false);
         SinglePartitionReadCommand command = SinglePartitionReadCommand.create(
-                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
+                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.none(), DataLimits.cqlLimits(3), Util.dk("0"), filter);
         assertRowsInResult(cfs, command, 1000, 1001, 1002);
 
         filter = slices(cfs, 1195, null, false);
         command = SinglePartitionReadCommand.create(
-                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
+                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.none(), DataLimits.cqlLimits(3), Util.dk("0"), filter);
         assertRowsInResult(cfs, command, 1195, 1196, 1197);
 
         filter = slices(cfs, null, 1996, true);
         command = SinglePartitionReadCommand.create(
-                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(1000), Util.dk("0"), filter);
+                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.none(), DataLimits.cqlLimits(1000), Util.dk("0"), filter);
         int[] expectedValues = new int[997];
         for (int i = 0, v = 1996; v >= 1000; i++, v--)
             expectedValues[i] = v;
@@ -501,22 +501,22 @@ public class KeyspaceTest extends CQLTester
 
         filter = slices(cfs, 1990, null, false);
         command = SinglePartitionReadCommand.create(
-                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
+                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.none(), DataLimits.cqlLimits(3), Util.dk("0"), filter);
         assertRowsInResult(cfs, command, 1990, 1991, 1992);
 
         filter = slices(cfs, null, null, true);
         command = SinglePartitionReadCommand.create(
-                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
+                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.none(), DataLimits.cqlLimits(3), Util.dk("0"), filter);
         assertRowsInResult(cfs, command, 1999, 1998, 1997);
 
         filter = slices(cfs, null, 9000, true);
         command = SinglePartitionReadCommand.create(
-                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
+                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.none(), DataLimits.cqlLimits(3), Util.dk("0"), filter);
         assertRowsInResult(cfs, command, 1999, 1998, 1997);
 
         filter = slices(cfs, 9000, null, false);
         command = SinglePartitionReadCommand.create(
-                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.cqlLimits(3), Util.dk("0"), filter);
+                cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.none(), DataLimits.cqlLimits(3), Util.dk("0"), filter);
         assertRowsInResult(cfs, command);
     }
 
diff --git a/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java b/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java
index b823f23dd9..5b54646f9e 100644
--- a/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java
+++ b/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java
@@ -166,7 +166,7 @@ public class ReadCommandVerbHandlerTest
                   metadata,
                   FBUtilities.nowInSeconds(),
                   ColumnFilter.all(metadata),
-                  RowFilter.NONE,
+                  RowFilter.none(),
                   DataLimits.NONE,
                   KEY,
                   new ClusteringIndexSliceFilter(Slices.ALL, false),
diff --git a/test/unit/org/apache/cassandra/db/ReadResponseTest.java b/test/unit/org/apache/cassandra/db/ReadResponseTest.java
index f8a19fda7d..edfba6cb19 100644
--- a/test/unit/org/apache/cassandra/db/ReadResponseTest.java
+++ b/test/unit/org/apache/cassandra/db/ReadResponseTest.java
@@ -258,7 +258,7 @@ public class ReadResponseTest
                   metadata,
                   FBUtilities.nowInSeconds(),
                   ColumnFilter.all(metadata),
-                  RowFilter.NONE,
+                  RowFilter.none(),
                   DataLimits.NONE,
                   metadata.partitioner.decorateKey(ByteBufferUtil.bytes(key)),
                   null,
diff --git a/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java b/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
index ab790dca50..fcd768bb5b 100644
--- a/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
+++ b/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
@@ -186,7 +186,7 @@ public class SinglePartitionSliceCommandTest
         ReadCommand cmd = SinglePartitionReadCommand.create(CFM_SLICES,
                                                             FBUtilities.nowInSeconds(),
                                                             ColumnFilter.all(CFM_SLICES),
-                                                            RowFilter.NONE,
+                                                            RowFilter.none(),
                                                             DataLimits.NONE,
                                                             key,
                                                             clusteringFilter);
@@ -247,7 +247,7 @@ public class SinglePartitionSliceCommandTest
         ReadCommand cmd = SinglePartitionReadCommand.create(metadata,
                                                             FBUtilities.nowInSeconds(),
                                                             columnFilter,
-                                                            RowFilter.NONE,
+                                                            RowFilter.none(),
                                                             DataLimits.NONE,
                                                             key,
                                                             sliceFilter);
@@ -475,7 +475,7 @@ public class SinglePartitionSliceCommandTest
         ReadCommand cmd = SinglePartitionReadCommand.create(metadata,
                                                             FBUtilities.nowInSeconds(),
                                                             columnFilter,
-                                                            RowFilter.NONE,
+                                                            RowFilter.none(),
                                                             DataLimits.NONE,
                                                             key,
                                                             sliceFilter);
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
index 3c0215432d..78de095c92 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
@@ -350,7 +350,7 @@ public class CompactionsTest
         for (FilteredPartition p : Util.getAll(Util.cmd(cfs).build()))
         {
             k.add(p.partitionKey());
-            final SinglePartitionReadCommand command = SinglePartitionReadCommand.create(cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.NONE, DataLimits.NONE, p.partitionKey(), new ClusteringIndexSliceFilter(Slices.ALL, false));
+            final SinglePartitionReadCommand command = SinglePartitionReadCommand.create(cfs.metadata(), FBUtilities.nowInSeconds(), ColumnFilter.all(cfs.metadata()), RowFilter.none(), DataLimits.NONE, p.partitionKey(), new ClusteringIndexSliceFilter(Slices.ALL, false));
             try (ReadExecutionController executionController = command.executionController();
                  PartitionIterator iterator = command.executeInternal(executionController))
             {
diff --git a/test/unit/org/apache/cassandra/db/filter/RowFilterTest.java b/test/unit/org/apache/cassandra/db/filter/RowFilterTest.java
index 333d3f8c9e..8952262e2b 100644
--- a/test/unit/org/apache/cassandra/db/filter/RowFilterTest.java
+++ b/test/unit/org/apache/cassandra/db/filter/RowFilterTest.java
@@ -63,7 +63,7 @@ public class RowFilterTest
         ColumnMetadata r = metadata.getColumn(new ColumnIdentifier("r", true));
 
         ByteBuffer one = Int32Type.instance.decompose(1);
-        RowFilter filter = RowFilter.NONE.withNewExpressions(new ArrayList<>());
+        RowFilter filter = RowFilter.none().withNewExpressions(new ArrayList<>());
         filter.add(s, Operator.NEQ, one);
         AtomicBoolean closed = new AtomicBoolean();
         UnfilteredPartitionIterator iter = filter.filter(new SingletonUnfilteredPartitionIterator(new UnfilteredRowIterator()
@@ -91,7 +91,7 @@ public class RowFilterTest
         Assert.assertFalse(iter.hasNext());
         Assert.assertTrue(closed.get());
 
-        filter = RowFilter.NONE.withNewExpressions(new ArrayList<>());
+        filter = RowFilter.none().withNewExpressions(new ArrayList<>());
         filter.add(r, Operator.NEQ, one);
         closed.set(false);
         iter = filter.filter(new SingletonUnfilteredPartitionIterator(new UnfilteredRowIterator()
diff --git a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBoundTest.java b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBoundTest.java
index 86df1cbe3e..d6fd93f603 100644
--- a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBoundTest.java
+++ b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBoundTest.java
@@ -245,7 +245,7 @@ public class UnfilteredRowIteratorWithLowerBoundTest
         SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(metadata,
                                                                            FBUtilities.nowInSeconds(),
                                                                            ColumnFilter.all(metadata),
-                                                                           RowFilter.NONE,
+                                                                           RowFilter.none(),
                                                                            DataLimits.NONE,
                                                                            key,
                                                                            filter);
@@ -260,4 +260,4 @@ public class UnfilteredRowIteratorWithLowerBoundTest
             return iter.lowerBound() != null;
         }
     }
-}
\ No newline at end of file
+}
diff --git a/test/unit/org/apache/cassandra/db/virtual/GossipInfoTableTest.java b/test/unit/org/apache/cassandra/db/virtual/GossipInfoTableTest.java
index a2c2a2bc7e..3863cbc189 100644
--- a/test/unit/org/apache/cassandra/db/virtual/GossipInfoTableTest.java
+++ b/test/unit/org/apache/cassandra/db/virtual/GossipInfoTableTest.java
@@ -77,7 +77,7 @@ public class GossipInfoTableTest extends CQLTester
 
             assertThat(resultSet.size()).isEqualTo(1);
             UntypedResultSet.Row row = resultSet.one();
-            assertThat(row.getColumns().size()).isEqualTo(64);
+            assertThat(row.getColumns().size()).isEqualTo(66);
 
             assertThat(endpoint).isNotNull();
             assertThat(localState).isNotNull();
diff --git a/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java b/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java
index c24690b8bf..c0573d61f7 100644
--- a/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java
+++ b/test/unit/org/apache/cassandra/dht/KeyCollisionTest.java
@@ -128,6 +128,12 @@ public class KeyCollisionTest
             return 0;
         }
 
+        @Override
+        public long getLongValue()
+        {
+            return token.longValue();
+        }
+
         @Override
         public ByteSource asComparableBytes(ByteComparable.Version version)
         {
diff --git a/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java b/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java
index c9dde2ae69..0ab2f82440 100644
--- a/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java
+++ b/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java
@@ -50,7 +50,6 @@ import static org.junit.Assert.fail;
 
 public class SecondaryIndexManagerTest extends CQLTester
 {
-
     @After
     public void after()
     {
diff --git a/test/unit/org/apache/cassandra/index/sai/IndexingSchemaLoader.java b/test/unit/org/apache/cassandra/index/sai/IndexingSchemaLoader.java
new file mode 100644
index 0000000000..62e6aa0860
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/IndexingSchemaLoader.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai;
+
+import java.util.HashMap;
+
+import org.apache.cassandra.SchemaLoader;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.marshal.DoubleType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.Indexes;
+import org.apache.cassandra.schema.TableMetadata;
+
+public class IndexingSchemaLoader extends SchemaLoader
+{
+    public static TableMetadata.Builder skinnySAITableMetadata(String ksName, String cfName)
+    {
+        TableMetadata.Builder builder =
+                TableMetadata.builder(ksName, cfName)
+                             .addPartitionKeyColumn("id", UTF8Type.instance)
+                             .addRegularColumn("first_name", UTF8Type.instance)
+                             .addRegularColumn("last_name", UTF8Type.instance)
+                             .addRegularColumn("age", Int32Type.instance)
+                             .addRegularColumn("height", Int32Type.instance)
+                             .addRegularColumn("timestamp", LongType.instance)
+                             .addRegularColumn("address", UTF8Type.instance)
+                             .addRegularColumn("score", DoubleType.instance)
+                             .addRegularColumn("comment", UTF8Type.instance)
+                             .addRegularColumn("comment_suffix_split", UTF8Type.instance)
+                             .addRegularColumn("/output/full-name/", UTF8Type.instance)
+                             .addRegularColumn("/data/output/id", UTF8Type.instance)
+                             .addRegularColumn("first_name_prefix", UTF8Type.instance);
+
+        Indexes.Builder indexes = Indexes.builder();
+
+        indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_first_name", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+        {{
+            put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+            put(IndexTarget.TARGET_OPTION_NAME, "first_name");
+        }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_last_name", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "last_name");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_age", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "age");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_timestamp", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "timestamp");
+
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_address", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "address");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_score", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "score");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_comment", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "comment");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_comment_suffix_split", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "comment_suffix_split");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_output_full_name", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "/output/full-name/");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_data_output_id", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "/data/output/id");
+                }}))
+                .add(IndexMetadata.fromSchemaMetadata(cfName + "_first_name_prefix", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+                {{
+                    put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                    put(IndexTarget.TARGET_OPTION_NAME, "first_name_prefix");
+                }}));
+
+        return builder.indexes(indexes.build());
+    }
+
+    public static TableMetadata.Builder clusteringSAITableMetadata(String ksName, String cfName)
+    {
+        return clusteringSAITableMetadata(ksName, cfName, "location", "age", "height", "score");
+    }
+
+    public static TableMetadata.Builder clusteringSAITableMetadata(String ksName, String cfName, String...indexedColumns)
+    {
+        Indexes.Builder indexes = Indexes.builder();
+        for (String indexedColumn : indexedColumns)
+        {
+            indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_" + indexedColumn, IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+            {{
+                put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+                put(IndexTarget.TARGET_OPTION_NAME, indexedColumn);
+            }}));
+        }
+
+        return TableMetadata.builder(ksName, cfName)
+                            .addPartitionKeyColumn("name", UTF8Type.instance)
+                            .addClusteringColumn("location", UTF8Type.instance)
+                            .addClusteringColumn("age", Int32Type.instance)
+                            .addRegularColumn("height", Int32Type.instance)
+                            .addRegularColumn("score", DoubleType.instance)
+                            .addStaticColumn("nickname", UTF8Type.instance)
+                            .indexes(indexes.build());
+    }
+
+    public static TableMetadata.Builder staticSAITableMetadata(String ksName, String cfName)
+    {
+        TableMetadata.Builder builder =
+                TableMetadata.builder(ksName, cfName)
+                             .addPartitionKeyColumn("sensor_id", Int32Type.instance)
+                             .addStaticColumn("sensor_type", UTF8Type.instance)
+                             .addClusteringColumn("date", LongType.instance)
+                             .addRegularColumn("value", DoubleType.instance)
+                             .addRegularColumn("variance", Int32Type.instance);
+
+        Indexes.Builder indexes = Indexes.builder();
+
+        indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_sensor_type", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+        {{
+            put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+            put(IndexTarget.TARGET_OPTION_NAME, "sensor_type");
+        }}));
+
+        indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_value", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+        {{
+            put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+            put(IndexTarget.TARGET_OPTION_NAME, "value");
+        }}));
+
+        indexes.add(IndexMetadata.fromSchemaMetadata(cfName + "_variance", IndexMetadata.Kind.CUSTOM, new HashMap<String, String>()
+        {{
+            put(IndexTarget.CUSTOM_INDEX_OPTION_NAME, StorageAttachedIndex.class.getName());
+            put(IndexTarget.TARGET_OPTION_NAME, "variance");
+        }}));
+
+        return builder.indexes(indexes.build());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/SAITester.java b/test/unit/org/apache/cassandra/index/sai/SAITester.java
new file mode 100644
index 0000000000..3710080795
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/SAITester.java
@@ -0,0 +1,360 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.index.sai;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import javax.management.AttributeNotFoundException;
+import javax.management.ObjectName;
+
+import org.junit.After;
+import org.junit.Rule;
+import org.junit.rules.TestRule;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+import com.datastax.driver.core.Session;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.cql3.statements.schema.IndexTarget;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeyFactory;
+import org.apache.cassandra.index.sai.utils.ResourceLeakDetector;
+import org.apache.cassandra.inject.Injections;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.service.snapshot.TableSnapshot;
+import org.apache.cassandra.utils.Throwables;
+import org.awaitility.Awaitility;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public abstract class SAITester extends CQLTester
+{
+    protected static final String CREATE_KEYSPACE_TEMPLATE = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = " +
+                                                             "{'class': 'SimpleStrategy', 'replication_factor': '1'}";
+
+    protected static final String CREATE_TABLE_TEMPLATE = "CREATE TABLE %s (id1 TEXT PRIMARY KEY, v1 INT, v2 TEXT) WITH compaction = " +
+                                                          "{'class' : 'SizeTieredCompactionStrategy', 'enabled' : false }";
+    protected static final String CREATE_INDEX_TEMPLATE = "CREATE CUSTOM INDEX IF NOT EXISTS ON %%s(%s) USING 'StorageAttachedIndex'";
+
+    private static Randomization random;
+
+    public static final ClusteringComparator EMPTY_COMPARATOR = new ClusteringComparator();
+
+    public static final PrimaryKeyFactory TEST_FACTORY = PrimaryKey.factory(EMPTY_COMPARATOR);
+
+    @Rule
+    public TestRule testRules = new ResourceLeakDetector();
+
+    @Rule
+    public FailureWatcher failureRule = new FailureWatcher();
+
+    @After
+    public void removeAllInjections()
+    {
+        Injections.deleteAll();
+    }
+
+    public static Randomization getRandom()
+    {
+        if (random == null)
+            random = new Randomization();
+        return random;
+    }
+
+    public static IndexContext createIndexContext(String name, AbstractType<?> validator)
+    {
+        return new IndexContext("test_ks",
+                                "test_cf",
+                                UTF8Type.instance,
+                                new ClusteringComparator(),
+                                ColumnMetadata.regularColumn("sai", "internal", name, validator),
+                                IndexTarget.Type.SIMPLE,
+                                IndexMetadata.fromSchemaMetadata(name, IndexMetadata.Kind.CUSTOM, null));
+    }
+
+    protected void waitForAssert(Runnable runnableAssert, long timeout, TimeUnit unit)
+    {
+        Awaitility.await().dontCatchUncaughtExceptions().atMost(timeout, unit).untilAsserted(runnableAssert::run);
+    }
+
+    protected boolean isIndexQueryable(String keyspace, String table)
+    {
+        ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+        for (Index index : cfs.indexManager.listIndexes())
+        {
+            if (!cfs.indexManager.isIndexQueryable(index))
+                return false;
+        }
+        return true;
+    }
+
+    protected Object getMBeanAttribute(ObjectName name, String attribute) throws Exception
+    {
+        return jmxConnection.getAttribute(name, attribute);
+    }
+
+    protected Object getMetricValue(ObjectName metricObjectName)
+    {
+        // lets workaround the fact that gauges have Value, but counters have Count
+        Object metricValue;
+        try
+        {
+            try
+            {
+                metricValue = getMBeanAttribute(metricObjectName, "Value");
+            }
+            catch (AttributeNotFoundException ignored)
+            {
+                metricValue = getMBeanAttribute(metricObjectName, "Count");
+            }
+        }
+        catch (Exception e)
+        {
+            throw new RuntimeException(e);
+        }
+        return metricValue;
+    }
+
+    public void waitForIndexQueryable()
+    {
+        waitForIndexQueryable(KEYSPACE, currentTable());
+    }
+
+    public void waitForIndexQueryable(String keyspace, String table)
+    {
+        waitForAssert(() -> assertTrue(isIndexQueryable(keyspace, table)), 60, TimeUnit.SECONDS);
+    }
+
+    protected void waitForCompactionsFinished()
+    {
+        waitForAssert(() -> assertEquals(0, getCompactionTasks()), 10, TimeUnit.SECONDS);
+    }
+
+    protected void waitForEquals(ObjectName name, long value)
+    {
+        waitForAssert(() -> assertEquals(value, ((Number) getMetricValue(name)).longValue()), 10, TimeUnit.SECONDS);
+    }
+
+    protected ObjectName objectName(String name, String keyspace, String table, String index, String type)
+    {
+        try
+        {
+            return new ObjectName(String.format("org.apache.cassandra.metrics:type=StorageAttachedIndex," +
+                                                "keyspace=%s,table=%s,index=%s,scope=%s,name=%s",
+                                                keyspace, table, index, type, name));
+        }
+        catch (Throwable ex)
+        {
+            throw Throwables.unchecked(ex);
+        }
+    }
+
+    protected ObjectName objectNameNoIndex(String name, String keyspace, String table, String type)
+    {
+        try
+        {
+            return new ObjectName(String.format("org.apache.cassandra.metrics:type=StorageAttachedIndex," +
+                                                "keyspace=%s,table=%s,scope=%s,name=%s",
+                                                keyspace, table, type, name));
+        }
+        catch (Throwable ex)
+        {
+            throw Throwables.unchecked(ex);
+        }
+    }
+
+    public String createTable(String query)
+    {
+        return createTable(KEYSPACE, query);
+    }
+
+    public UntypedResultSet execute(String query, Object... values) throws Throwable
+    {
+        return executeFormattedQuery(formatQuery(query), values);
+    }
+
+    public Session sessionNet()
+    {
+        return sessionNet(getDefaultVersion());
+    }
+
+    public void flush(String keyspace, String table)
+    {
+        ColumnFamilyStore store = Keyspace.open(keyspace).getColumnFamilyStore(table);
+        if (store != null)
+            store.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
+    }
+
+    public void compact(String keyspace, String table)
+    {
+        ColumnFamilyStore store = Keyspace.open(keyspace).getColumnFamilyStore(table);
+        if (store != null)
+            store.forceMajorCompaction();
+    }
+
+    protected void truncate(boolean snapshot)
+    {
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+        if (snapshot)
+            cfs.truncateBlocking();
+        else
+            cfs.truncateBlockingWithoutSnapshot();
+    }
+
+    protected int getCompactionTasks()
+    {
+        return CompactionManager.instance.getActiveCompactions() + CompactionManager.instance.getPendingTasks();
+    }
+
+    protected int snapshot(String snapshotName)
+    {
+        ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(currentTable());
+        TableSnapshot snapshot = cfs.snapshot(snapshotName);
+        return snapshot.getDirectories().size();
+    }
+
+    protected List<String> restore(ColumnFamilyStore cfs, Directories.SSTableLister lister)
+    {
+        File dataDirectory = cfs.getDirectories().getDirectoryForNewSSTables();
+
+        List<String> fileNames = new ArrayList<>();
+        for (File file : lister.listFiles())
+        {
+            if (file.tryMove(new File(dataDirectory.absolutePath() + File.pathSeparator() + file.name())))
+            {
+                fileNames.add(file.name());
+            }
+        }
+        cfs.loadNewSSTables();
+        return fileNames;
+    }
+
+    public static class Randomization
+    {
+        private final long seed;
+        private final Random random;
+
+        Randomization()
+        {
+            seed = Long.getLong("cassandra.test.random.seed", System.nanoTime());
+            random = new Random(seed);
+        }
+
+        public void printSeedOnFailure()
+        {
+            System.err.println("Randomized test failed. To rerun test use -Dcassandra.test.random.seed=" + seed);
+        }
+
+        public int nextInt()
+        {
+            return random.nextInt();
+        }
+
+        public int nextIntBetween(int minValue, int maxValue)
+        {
+            return RandomInts.randomIntBetween(random, minValue, maxValue);
+        }
+
+        public long nextLong()
+        {
+            return random.nextLong();
+        }
+
+        public short nextShort()
+        {
+            return (short)random.nextInt(Short.MAX_VALUE + 1);
+        }
+
+        public byte nextByte()
+        {
+            return (byte)random.nextInt(Byte.MAX_VALUE + 1);
+        }
+
+        public BigInteger nextBigInteger(int minNumBits, int maxNumBits)
+        {
+            return new BigInteger(RandomInts.randomIntBetween(random, minNumBits, maxNumBits), random);
+        }
+
+        public BigDecimal nextBigDecimal(int minUnscaledValue, int maxUnscaledValue, int minScale, int maxScale)
+        {
+            return BigDecimal.valueOf(RandomInts.randomIntBetween(random, minUnscaledValue, maxUnscaledValue),
+                                      RandomInts.randomIntBetween(random, minScale, maxScale));
+        }
+
+        public float nextFloat()
+        {
+            return random.nextFloat();
+        }
+
+        public double nextDouble()
+        {
+            return random.nextDouble();
+        }
+
+        public String nextAsciiString(int minLength, int maxLength)
+        {
+            return RandomStrings.randomAsciiOfLengthBetween(random, minLength, maxLength);
+        }
+
+        public String nextTextString(int minLength, int maxLength)
+        {
+            return RandomStrings.randomRealisticUnicodeOfLengthBetween(random, minLength, maxLength);
+        }
+
+        public boolean nextBoolean()
+        {
+            return random.nextBoolean();
+        }
+
+        public void nextBytes(byte[] bytes)
+        {
+            random.nextBytes(bytes);
+        }
+    }
+
+    public static class FailureWatcher extends TestWatcher
+    {
+        @Override
+        protected void failed(Throwable e, Description description)
+        {
+            if (random != null)
+                random.printSeedOnFailure();
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/AbstractQueryTester.java b/test/unit/org/apache/cassandra/index/sai/cql/AbstractQueryTester.java
new file mode 100644
index 0000000000..41a3b769a5
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/AbstractQueryTester.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.index.sai.SAITester;
+import org.apache.cassandra.index.sai.plan.StorageAttachedIndexSearcher;
+import org.apache.cassandra.inject.Injections;
+
+import static org.apache.cassandra.inject.InvokePointBuilder.newInvokePoint;
+
+@RunWith(Parameterized.class)
+public class AbstractQueryTester extends SAITester
+{
+    public static final Injections.Counter INDEX_QUERY_COUNTER = Injections.newCounter("IndexQueryCounter")
+                                                                              .add(newInvokePoint().onClass(StorageAttachedIndexSearcher.class).onMethod("search"))
+                                                                              .build();
+
+    @Parameterized.Parameter(0)
+    public BaseDataModel dataModel;
+    @Parameterized.Parameter(1)
+    public List<IndexQuerySupport.BaseQuerySet> sets;
+
+    protected BaseDataModel.Executor executor;
+
+    @Before
+    public void setup() throws Throwable
+    {
+        requireNetwork();
+
+        schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", BaseDataModel.KEYSPACE));
+
+        Injections.inject(INDEX_QUERY_COUNTER);
+
+        executor = new SingleNodeExecutor(this, INDEX_QUERY_COUNTER);
+    }
+
+    @SuppressWarnings("unused")
+    @Parameterized.Parameters(name = "{0}")
+    public static List<Object[]> params() throws Throwable
+    {
+        List<Object[]> scenarios = new LinkedList<>();
+
+        scenarios.add(new Object[]{ new BaseDataModel(BaseDataModel.NORMAL_COLUMNS, BaseDataModel.NORMAL_COLUMN_DATA), IndexQuerySupport.BASE_QUERY_SETS });
+
+        scenarios.add(new Object[]{ new BaseDataModel.CompoundKeyDataModel(BaseDataModel.NORMAL_COLUMNS, BaseDataModel.NORMAL_COLUMN_DATA), IndexQuerySupport.BASE_QUERY_SETS });
+
+        scenarios.add(new Object[]{ new BaseDataModel.CompoundKeyWithStaticsDataModel(BaseDataModel.STATIC_COLUMNS, BaseDataModel.STATIC_COLUMN_DATA), IndexQuerySupport.STATIC_QUERY_SETS });
+
+        scenarios.add(new Object[]{ new BaseDataModel.CompositePartitionKeyDataModel(BaseDataModel.NORMAL_COLUMNS, BaseDataModel.NORMAL_COLUMN_DATA),
+                                    ImmutableList.builder().addAll(IndexQuerySupport.BASE_QUERY_SETS).addAll(IndexQuerySupport.COMPOSITE_PARTITION_QUERY_SETS).build()});
+
+        return scenarios;
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/BaseDataModel.java b/test/unit/org/apache/cassandra/index/sai/cql/BaseDataModel.java
new file mode 100644
index 0000000000..bd61fc4273
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/BaseDataModel.java
@@ -0,0 +1,397 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.ForwardingList;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+
+import org.apache.cassandra.cql3.CQL3Type;
+import org.apache.cassandra.utils.Pair;
+
+class BaseDataModel
+{
+    public static String KEYSPACE = "sai_query_keyspace";
+
+    public static String SIMPLE_SELECT_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? LIMIT ?";
+    public static String SIMPLE_SELECT_WITH_FILTERING_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? LIMIT ? ALLOW FILTERING";
+    public static String TWO_CLAUSE_AND_QUERY_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? AND %s %s ? LIMIT ? ALLOW FILTERING";
+    public static String TWO_CLAUSE_AND_QUERY_FILTERING_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? AND %s %s ? LIMIT ? ALLOW FILTERING";
+    public static String THREE_CLAUSE_AND_QUERY_FILTERING_TEMPLATE = "SELECT %s FROM %%s WHERE %s %s ? AND %s %s ? AND %s %s ? LIMIT ? ALLOW FILTERING";
+
+    public static String ASCII_COLUMN = "abbreviation";
+    public static String BIGINT_COLUMN = "gdp";
+    public static String BOOLEAN_COLUMN = "active";
+    public static String DATE_COLUMN = "visited";
+    public static String DOUBLE_COLUMN = "area_sq_miles";
+    public static String FLOAT_COLUMN = "murder_rate";
+    public static String INET_COLUMN = "ip";
+    public static String INT_COLUMN = "population";
+    public static String SMALLINT_COLUMN = "murders_per_year";
+    public static String TINYINT_COLUMN = "tiny_murders_per_year";
+    public static String TEXT_COLUMN = "name";
+    public static String TIME_COLUMN = "avg_dmv_wait";
+    public static String TIMESTAMP_COLUMN = "visited_timestamp";
+    public static String UUID_COLUMN = "id";
+    public static String TIMEUUID_COLUMN = "temporal_id";
+    public static String NON_INDEXED_COLUMN = "non_indexed";
+
+    public static int DEFAULT_TTL_SECONDS = 10;
+
+    public static List<Pair<String, String>> NORMAL_COLUMNS =
+            ImmutableList.<Pair<String, String>>builder()
+                    .add(Pair.create(ASCII_COLUMN, CQL3Type.Native.ASCII.toString()))
+                    .add(Pair.create(BIGINT_COLUMN, CQL3Type.Native.BIGINT.toString()))
+                    .add(Pair.create(BOOLEAN_COLUMN, CQL3Type.Native.BOOLEAN.toString()))
+                    .add(Pair.create(DATE_COLUMN, CQL3Type.Native.DATE.toString()))
+                    .add(Pair.create(DOUBLE_COLUMN, CQL3Type.Native.DOUBLE.toString()))
+                    .add(Pair.create(FLOAT_COLUMN, CQL3Type.Native.FLOAT.toString()))
+                    .add(Pair.create(INET_COLUMN, CQL3Type.Native.INET.toString()))
+                    .add(Pair.create(INT_COLUMN, CQL3Type.Native.INT.toString()))
+                    .add(Pair.create(SMALLINT_COLUMN, CQL3Type.Native.SMALLINT.toString()))
+                    .add(Pair.create(TINYINT_COLUMN, CQL3Type.Native.TINYINT.toString()))
+                    .add(Pair.create(TEXT_COLUMN, CQL3Type.Native.TEXT.toString()))
+                    .add(Pair.create(TIME_COLUMN, CQL3Type.Native.TIME.toString()))
+                    .add(Pair.create(TIMESTAMP_COLUMN, CQL3Type.Native.TIMESTAMP.toString()))
+                    .add(Pair.create(UUID_COLUMN, CQL3Type.Native.UUID.toString()))
+                    .add(Pair.create(TIMEUUID_COLUMN, CQL3Type.Native.TIMEUUID.toString()))
+                    .add(Pair.create(NON_INDEXED_COLUMN, CQL3Type.Native.INT.toString()))
+                    .build();
+
+    public static List<String> NORMAL_COLUMN_DATA =
+            ImmutableList.<String>builder()
+                    .add("'AK',   500000000,  true, '2009-07-15', 570640.95,  7.7,   '158.145.20.64',   737709,  164,  16,        'Alaska', '00:18:20', '2009-07-15T00:00:00', e37394dc-d17b-11e8-a8d5-f2801f1b9fd1, acfe5ada-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'AL',  1000000000,  true, '2011-09-13',  50645.33,  7.0,   '206.16.212.91',  4853875,   57,   5,       'Alabama', '01:04:00', '2011-09-13T00:00:00', b7373af6-d7c1-45ae-b145-5bf4b5cdd00c, c592c37e-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'AR',  2000000000, false, '2013-06-17', 113594.08,  5.5,  '170.94.194.134',  2977853,   99,   9,      'Arkansas', '00:55:23', '2013-06-17T00:00:00', a0daaeb4-c8a2-4c68-9899-e32d08238550, cfaae67a-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'CA',  3000000000,  true, '2012-06-17', 155779.22,  4.8,    '67.157.98.46', 38993940, 1861, 117,    'California', '01:30:45', '2012-06-17T00:00:00', 96232af0-0af7-438b-9049-c5a5a944ff93, d7e80692-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'DE',  4000000000, false, '2013-06-17',   1948.54,  6.7,   '167.21.128.20',   944076,   63,   6,      'Delaware', '00:23:45', '2013-06-17T00:00:00', b2a0a879-5223-40d2-9671-775ee209b6f2, dd10a5b6-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'ID',  4500000000, false, '2015-06-18',  82643.12,  1.8,   '164.165.67.10',  1652828,   30,   3,         'Idaho', '00:18:45', '2015-06-18T00:00:00', c6eec0b0-0eef-40e8-ac38-3a82110443e4, e2788780-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'KY',  4750000000, false, '2018-03-12',  39486.34,  4.7,  '205.204.196.64',  4424611,  209,  20,      'Kentucky', '00:45:00', '2018-03-12T00:00:00', 752355f8-405b-4d94-88f3-9992cda30f1e, e7c4e1d4-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'LA',  4800000000,  true, '2013-06-10',  43203.90, 10.2,  '204.196.242.71',  4668960,  474,  47,     'Louisiana', '00:56:07', '2013-06-10T00:00:00', 17be691a-c1a4-4467-a4ad-64605c74fb1c, ee6136d2-d17c-11e8-a8d5-f2801f1b9fd1, 1")
+                    .add("'MA',  5000000000,  true,  '2010-07-04',   7800.06,  1.9,   '170.63.206.57',  6784240,  126,  12, 'Massachusetts', '01:01:34', '2010-07-04T00:00:00', e8a3c287-78cf-46b5-b554-42562e7dcfb3, f57a3b62-d17c-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'MI',  6000000000, false, '2011-09-13',  56538.90,  5.8,    '23.72.184.64',  9917715,  571,  57,      'Michigan', '00:43:09', '2011-09-13T00:00:00', a0daaeb4-c8a2-4c68-9899-e32d08238550, 0497b886-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'MS',  7000000000,  true, '2013-06-17',  46923.27,  5.3,   '192.251.58.38',  2989390,  159,  15,   'Mississippi', '01:04:23', '2013-06-17T00:00:00', 96232af0-0af7-438b-9049-c5a5a944ff93, 0b0205e6-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'TN',  8000000000, false, '2018-03-10',  41234.90,  6.1, '170.141.221.177',  6595056,  402,  40,     'Tennessee', '00:39:45', '2018-03-10T00:00:00', b2a0a879-5223-40d2-9671-775ee209b6f2, 105dc746-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'TX',  9000000000,  true, '2014-06-17', 261231.71,  4.7,   '204.66.40.181', 27429639, 1276, 107,         'Texas', '00:38:13', '2014-06-17T00:00:00', c6eec0b0-0eef-40e8-ac38-3a82110443e4, 155b6bcc-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'UT',  9250000000,  true, '2014-06-20',  82169.62,  1.8,   '204.113.13.48',  2990632,   54,   5,          'Utah', '00:25:00', '2014-06-20T00:00:00', 752355f8-405b-4d94-88f3-9992cda30f1e, 1a267c50-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'VA',  9500000000,  true, '2018-06-19',  39490.09,  4.6,  '152.130.96.221',  8367587,  383,  38,      'Virginia', '00:43:07', '2018-06-19T00:00:00', 17be691a-c1a4-4467-a4ad-64605c74fb1c, 1fc81a4c-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .add("'WY', 10000000000, false, '2015-06-17',  97093.14,  2.7,  '192.146.215.91',   586107,   57,   5,       'Wyoming', '00:15:50', '2015-06-17T00:00:00', e8a3c287-78cf-46b5-b554-42562e7dcfb3, 2576612e-d17d-11e8-a8d5-f2801f1b9fd1, 2")
+                    .build();
+
+    public static String STATIC_INT_COLUMN = "entered";
+
+    public static List<Pair<String, String>> STATIC_COLUMNS =
+            ImmutableList.<Pair<String, String>>builder().add(Pair.create(STATIC_INT_COLUMN, CQL3Type.Native.INT.toString() + " static"))
+                                                         .addAll(NORMAL_COLUMNS).build();
+
+    public static List<String> STATIC_COLUMN_DATA = ImmutableList.of("1819, " + NORMAL_COLUMN_DATA.get(0),
+                                                       "1819, " + NORMAL_COLUMN_DATA.get(1),
+                                                       "1850, " + NORMAL_COLUMN_DATA.get(2),
+                                                       "1850, " + NORMAL_COLUMN_DATA.get(3),
+                                                       "1910, " + NORMAL_COLUMN_DATA.get(4),
+                                                       "1910, " + NORMAL_COLUMN_DATA.get(5),
+                                                       "1792, " + NORMAL_COLUMN_DATA.get(6),
+                                                       "1792, " + NORMAL_COLUMN_DATA.get(7),
+                                                       "1788, " + NORMAL_COLUMN_DATA.get(8),
+                                                       "1788, " + NORMAL_COLUMN_DATA.get(9),
+                                                       "1817, " + NORMAL_COLUMN_DATA.get(10),
+                                                       "1817, " + NORMAL_COLUMN_DATA.get(11),
+                                                       "1896, " + NORMAL_COLUMN_DATA.get(12),
+                                                       "1896, " + NORMAL_COLUMN_DATA.get(13),
+                                                       "1845, " + NORMAL_COLUMN_DATA.get(14),
+                                                       "1845, " + NORMAL_COLUMN_DATA.get(15));
+
+    private static final AtomicInteger seq = new AtomicInteger();
+
+    private final String columnNames;
+    private final List<String> rows;
+
+    protected final Set<String> skipColumns = Sets.newHashSet(NON_INDEXED_COLUMN, BOOLEAN_COLUMN);
+    protected final List<Pair<String, String>> columns;
+    protected final String indexedTable = "table_" + seq.getAndIncrement();
+    protected final String nonIndexedTable = "table_" + seq.getAndIncrement();
+    protected String tableOptions = "";
+    protected List<Pair<String, String>> keyColumns;
+    protected String primaryKey;
+    protected List<String> keys;
+
+    public BaseDataModel(List<Pair<String, String>> columns, List<String> rows)
+    {
+        this.keyColumns = ImmutableList.of(Pair.create("p", "int"));
+        this.primaryKey = keyColumns.stream().map(pair -> pair.left).collect(Collectors.joining(", "));
+
+        this.columns = columns;
+        this.columnNames = columns.stream().map(pair -> pair.left).collect(Collectors.joining(", "));
+        this.rows = rows;
+
+        this.keys = new SimplePrimaryKeyList(rows.size());
+    }
+
+    public List<Pair<String, String>> keyColumns()
+    {
+        return keyColumns;
+    }
+
+    public void createTables(Executor tester)
+    {
+        String keyColumnDefs = keyColumns.stream().map(column -> column.left + ' ' + column.right).collect(Collectors.joining(", "));
+        String normalColumnDefs = columns.stream().map(column -> column.left + ' ' + column.right).collect(Collectors.joining(", "));
+
+        String template = "CREATE TABLE %s (%s, %s, PRIMARY KEY (%s))" + tableOptions;
+        tester.createTable(String.format(template, KEYSPACE + "." + indexedTable, keyColumnDefs, normalColumnDefs, primaryKey));
+        tester.createTable(String.format(template, KEYSPACE + "." + nonIndexedTable, keyColumnDefs, normalColumnDefs, primaryKey));
+    }
+
+    public void createIndexes(Executor tester) throws Throwable
+    {
+        String template = "CREATE CUSTOM INDEX ndi_%s_index_%s ON %%s (%s) USING 'StorageAttachedIndex'";
+
+        for (Pair<String, String> column : columns)
+        {
+            if (!skipColumns.contains(column.left))
+            {
+                executeLocalIndexed(tester, String.format(template, column.left, indexedTable, column.left));
+                tester.waitForIndexQueryable(KEYSPACE, indexedTable);
+            }
+        }
+    }
+
+    public void insertRows(Executor tester) throws Throwable
+    {
+        String template = "INSERT INTO %%s (%s, %s) VALUES (%s, %s)";
+
+        for (int i = 0; i < keys.size(); i++)
+        {
+            executeLocal(tester, String.format(template, primaryKey, columnNames, keys.get(i), rows.get(i)));
+        }
+    }
+
+    public void insertRowsWithTTL(Executor tester) throws Throwable
+    {
+        String template = "INSERT INTO %%s (%s, %s) VALUES (%s, %s)%s";
+
+        for (int i = 0; i < keys.size(); i++)
+        {
+            String ttl = deletable().contains(i) ? " USING TTL " + DEFAULT_TTL_SECONDS : "";
+            executeLocal(tester, String.format(template, primaryKey, columnNames, keys.get(i), rows.get(i), ttl));
+        }
+    }
+
+    public void executeLocal(Executor tester, String query, Object... values) throws Throwable
+    {
+        tester.executeLocal(formatIndexedQuery(query), values);
+        tester.executeLocal(formatNonIndexedQuery(query), values);
+    }
+
+    public void executeLocalIndexed(Executor tester, String query, Object... values) throws Throwable
+    {
+        tester.executeLocal(formatIndexedQuery(query), values);
+    }
+
+    public List<Object> executeIndexed(Executor tester, String query, int fetchSize, Object... values) throws Throwable
+    {
+        return tester.executeRemote(formatIndexedQuery(query), fetchSize, values);
+    }
+
+    public List<Object> executeNonIndexed(Executor tester, String query, int fetchSize, Object... values) throws Throwable
+    {
+        return tester.executeRemote(formatNonIndexedQuery(query), fetchSize, values);
+    }
+
+    protected Set<Integer> deletable()
+    {
+        return Sets.newHashSet(3, 7, 9, 12);
+    }
+
+    private String formatIndexedQuery(String query)
+    {
+        return String.format(query, KEYSPACE + "." + indexedTable);
+    }
+
+    private String formatNonIndexedQuery(String query)
+    {
+        return String.format(query, KEYSPACE + "." + nonIndexedTable);
+    }
+
+    @Override
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this).add("primaryKey", primaryKey).toString();
+    }
+
+    static class CompoundKeyWithStaticsDataModel extends CompoundKeyDataModel
+    {
+        public CompoundKeyWithStaticsDataModel(List<Pair<String, String>> columns, List<String> rows)
+        {
+            super(columns, rows);
+
+            this.keys = new CompoundPrimaryKeyList(rows.size(), 2);
+        }
+
+        @Override
+        public void insertRows(Executor tester) throws Throwable
+        {
+            super.insertRows(tester);
+
+            executeLocal(tester, String.format("INSERT INTO %%s (p, %s) VALUES(100, 2019)", STATIC_INT_COLUMN)); // static only
+        }
+
+        @Override
+        protected Set<Integer> deletable()
+        {
+            return Sets.newHashSet(4, 8, 12, 13, 100);
+        }
+    }
+
+    static class CompositePartitionKeyDataModel extends BaseDataModel
+    {
+        public CompositePartitionKeyDataModel(List<Pair<String, String>> columns, List<String> rows)
+        {
+            super(columns, rows);
+
+            this.keyColumns = ImmutableList.of(Pair.create("p1", "int"), Pair.create("p2", "int"));
+            this.primaryKey = keyColumns.stream().map(pair -> pair.left).collect(Collectors.joining(", "));
+            this.keys = new CompoundPrimaryKeyList(rows.size(), 2);
+        }
+
+        @Override
+        public void createTables(Executor tester)
+        {
+            String keyColumnDefs = keyColumns.stream().map(column -> column.left + ' ' + column.right).collect(Collectors.joining(", "));
+            String normalColumnDefs = columns.stream().map(column -> column.left + ' ' + column.right).collect(Collectors.joining(", "));
+
+            String template = "CREATE TABLE %s (%s, %s, PRIMARY KEY ((%s)))" + tableOptions;
+            tester.createTable(String.format(template, KEYSPACE + "." + indexedTable, keyColumnDefs, normalColumnDefs, primaryKey));
+            tester.createTable(String.format(template, KEYSPACE + "." + nonIndexedTable, keyColumnDefs, normalColumnDefs, primaryKey));
+        }
+
+        @Override
+        public void createIndexes(Executor tester) throws Throwable
+        {
+            super.createIndexes(tester);
+            String template = "CREATE CUSTOM INDEX ndi_%s_index_%s ON %%s (%s) USING 'StorageAttachedIndex'";
+
+            for (Pair<String, String> column : keyColumns)
+            {
+                if (!skipColumns.contains(column.left))
+                {
+                    executeLocalIndexed(tester, String.format(template, column.left, indexedTable, column.left));
+                    tester.waitForIndexQueryable(KEYSPACE, indexedTable);
+                }
+            }
+        }
+
+        @Override
+        public void insertRows(Executor tester) throws Throwable
+        {
+            super.insertRows(tester);
+        }
+
+        @Override
+        protected Set<Integer> deletable()
+        {
+            // already overwrites {@code deleteRows()}
+            return Collections.emptySet();
+        }
+    }
+
+    static class CompoundKeyDataModel extends BaseDataModel
+    {
+        public CompoundKeyDataModel(List<Pair<String, String>> columns, List<String> rows)
+        {
+            super(columns, rows);
+
+            this.keyColumns = ImmutableList.of(Pair.create("p", "int"), Pair.create("c", "int"));
+            this.primaryKey = keyColumns.stream().map(pair -> pair.left).collect(Collectors.joining(", "));
+            this.keys = new CompoundPrimaryKeyList(rows.size(), 1);
+        }
+
+    }
+
+    static class SimplePrimaryKeyList extends ForwardingList<String>
+    {
+        private final List<String> primaryKeys;
+
+        SimplePrimaryKeyList(int rows)
+        {
+            this.primaryKeys = IntStream.range(0, rows).mapToObj(String::valueOf).collect(Collectors.toList());
+        }
+
+        @Override
+        protected List<String> delegate()
+        {
+            return primaryKeys;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("SimplePrimaryKeyList[rows: %d]", primaryKeys.size());
+        }
+    }
+
+    static class CompoundPrimaryKeyList extends ForwardingList<String>
+    {
+        private final List<String> primaryKeys;
+        private final int rowsPerPartition;
+
+        CompoundPrimaryKeyList(int rows, int rowsPerPartition)
+        {
+            this.primaryKeys = IntStream.range(0, rows).mapToObj(v -> v / rowsPerPartition + ", " + v % rowsPerPartition).collect(Collectors.toList());
+            this.rowsPerPartition = rowsPerPartition;
+        }
+
+        @Override
+        protected List<String> delegate()
+        {
+            return primaryKeys;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("CompoundPrimaryKeyList[rows: %d, partition size: %d]", primaryKeys.size(), rowsPerPartition);
+        }
+    }
+
+    interface Executor
+    {
+        void createTable(String statement);
+
+        void waitForIndexQueryable(String keyspace, String table);
+
+        void executeLocal(String query, Object...values) throws Throwable;
+
+        List<Object> executeRemote(String query, int fetchSize, Object...values) throws Throwable;
+
+        void counterReset();
+
+        long getCounter();
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/BooleanTypeTest.java b/test/unit/org/apache/cassandra/index/sai/cql/BooleanTypeTest.java
new file mode 100644
index 0000000000..3131414418
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/BooleanTypeTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.SAITester;
+
+import static org.junit.Assert.assertEquals;
+
+public class BooleanTypeTest extends SAITester
+{
+    @Test
+    public void test() throws Throwable
+    {
+        createTable("CREATE TABLE %s (id text PRIMARY KEY, val boolean)");
+
+        createIndex("CREATE CUSTOM INDEX ON %s(val) USING 'StorageAttachedIndex'");
+        waitForIndexQueryable();
+
+        execute("INSERT INTO %s (id, val) VALUES ('0', false)");
+        execute("INSERT INTO %s (id, val) VALUES ('1', true)");
+        execute("INSERT INTO %s (id, val) VALUES ('2', true)");
+
+        assertEquals(2, execute("SELECT id FROM %s WHERE val = true").size());
+        assertEquals(1, execute("SELECT id FROM %s WHERE val = false").size());
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/ClusteringKeyIndexTest.java b/test/unit/org/apache/cassandra/index/sai/cql/ClusteringKeyIndexTest.java
new file mode 100644
index 0000000000..58443fe0fb
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/ClusteringKeyIndexTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.SAITester;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+public class ClusteringKeyIndexTest extends SAITester
+{
+    @Before
+    public void createTableAndIndex()
+    {
+        createTable("CREATE TABLE %s (pk1 int, pk2 text, val int, PRIMARY KEY((pk1), pk2)) WITH CLUSTERING ORDER BY (pk2 DESC)");
+        createIndex("CREATE CUSTOM INDEX pk2_idx ON %s(pk2) USING 'StorageAttachedIndex'");
+
+        disableCompaction();
+    }
+
+    private void insertData1() throws Throwable
+    {
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (1, '1', 1)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (2, '2', 2)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (3, '3', 3)");
+    }
+
+    private void insertData2() throws Throwable
+    {
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (4, '4', 4)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (5, '5', 5)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (6, '6', 6)");
+    }
+
+    @Test
+    public void queryFromMemtable() throws Throwable
+    {
+        insertData1();
+        insertData2();
+        runQueries();
+    }
+
+    private Object[] expectedRow(int index)
+    {
+        return row(index, Integer.toString(index), index);
+    }
+
+    private void runQueries() throws Throwable
+    {
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 = 2"), expectedRow(2));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk2 = '2'"), expectedRow(2));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 = -1 AND pk2 = '2'"));
+
+        assertThatThrownBy(()->execute("SELECT * FROM %s WHERE pk1 = -1 AND val = 2")).hasMessageContaining("use ALLOW FILTERING");
+
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/CollectionIndexingTest.java b/test/unit/org/apache/cassandra/index/sai/cql/CollectionIndexingTest.java
new file mode 100644
index 0000000000..30709aa612
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/CollectionIndexingTest.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql;
+
+import java.util.Arrays;
+import java.util.HashMap;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.SAITester;
+
+import static org.junit.Assert.assertEquals;
+
+// This test is primarily handling edge conditions, error conditions
+// and basic functionality. Comprehensive type testing of collections
+// is in the cql/types/collections package
+public class CollectionIndexingTest extends SAITester
+{
+    private static final String ALLOW_FILTERING_ERROR = "Cannot execute this query as it might involve data filtering " +
+                                                        "and thus may have unpredictable performance. If you want to " +
+                                                        "execute this query despite the performance unpredictability, " +
+                                                        "use ALLOW FILTERING";
+    @Before
+    public void setup() throws Throwable
+    {
+        requireNetwork();
+    }
+
+    @Test
+    public void indexMap() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(value) USING 'StorageAttachedIndex'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value CONTAINS 'v1'").size());
+    }
+
+    @Test
+    public void indexMapKeys() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(KEYS(value)) USING 'StorageAttachedIndex'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value CONTAINS KEY 1").size());
+    }
+
+    @Test
+    public void indexMapValues() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(VALUES(value)) USING 'StorageAttachedIndex'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value CONTAINS 'v1'").size());
+    }
+
+    @Test
+    public void indexMapEntries() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(ENTRIES(value)) USING 'StorageAttachedIndex'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value[1] = 'v1'").size());
+        assertEquals(1, execute("SELECT * FROM %s WHERE value[1] = 'v1' AND value[2] = 'v2' ALLOW FILTERING").size());
+    }
+
+    @Test
+    public void indexFrozenList() throws Throwable
+    {
+        createPopulatedFrozenList("CREATE CUSTOM INDEX ON %s(FULL(value)) USING 'StorageAttachedIndex'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value = ?", Arrays.asList(1, 2, 3)).size());
+    }
+
+    @Test
+    public void indexFrozenMap() throws Throwable
+    {
+        createPopulatedFrozenMap("CREATE CUSTOM INDEX ON %s(FULL(value)) USING 'StorageAttachedIndex'");
+        assertEquals(1, execute("SELECT * FROM %s WHERE value = ?", new HashMap<Integer, String>() {{
+            put(1, "v1");
+            put(2, "v2");
+        }}).size());
+
+    }
+
+    @Test
+    public void indexFrozenMapQueryKeys() throws Throwable
+    {
+        createPopulatedFrozenMap("CREATE CUSTOM INDEX ON %s(FULL(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains key 1");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value contains key 1 ALLOW FILTERING").size());
+    }
+
+    @Test
+    public void indexFrozenMapQueryValues() throws Throwable
+    {
+        createPopulatedFrozenMap("CREATE CUSTOM INDEX ON %s(FULL(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains 'v1'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value contains 'v1' ALLOW FILTERING").size());
+    }
+
+    @Test
+    public void indexFrozenMapQueryEntries() throws Throwable
+    {
+        createPopulatedFrozenMap("CREATE CUSTOM INDEX ON %s(FULL(value)) USING 'StorageAttachedIndex'");
+        assertInvalidMessage("Map-entry equality predicates on frozen map column value are not supported",
+                "SELECT * FROM %s WHERE value[1] = 'v1'");
+    }
+
+    @Test
+    public void indexMapEntriesQueryEq() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(ENTRIES(value)) USING 'StorageAttachedIndex'");
+        assertInvalidMessage("Collection column 'value' (map<int, text>) cannot be restricted by a '=' relation",
+                "SELECT * FROM %s WHERE value = ?", Arrays.asList(1, 2));
+    }
+
+    @Test
+    public void indexMapEntriesQueryKeys() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(ENTRIES(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains key 1");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value contains key 1 ALLOW FILTERING").size());
+    }
+
+    @Test
+    public void indexMapEntriesQueryValues() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(ENTRIES(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains 'v1'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value contains 'v1' ALLOW FILTERING").size());
+    }
+
+    @Test
+    public void indexMapKeysQueryEq() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(KEYS(value)) USING 'StorageAttachedIndex'");
+        assertInvalidMessage("Collection column 'value' (map<int, text>) cannot be restricted by a '=' relation",
+                "SELECT * FROM %s WHERE value = ?", Arrays.asList(1, 2));
+    }
+
+    @Test
+    public void indexMapKeysQueryValues() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(KEYS(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains 'v1'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value contains 'v1' ALLOW FILTERING").size());
+    }
+
+    @Test
+    public void indexMapKeysQueryEntries() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(KEYS(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value[1] = 'v1'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value[1] = 'v1' ALLOW FILTERING").size());
+    }
+
+    @Test
+    public void indexMapValuesQueryEq() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(VALUES(value)) USING 'StorageAttachedIndex'");
+        assertInvalidMessage("Collection column 'value' (map<int, text>) cannot be restricted by a '=' relation",
+                "SELECT * FROM %s WHERE value = ?", Arrays.asList(1, 2));
+    }
+
+    @Test
+    public void indexMapValuesQueryKeys() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(VALUES(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value contains key 1");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value contains key 1 ALLOW FILTERING").size());
+    }
+
+    @Test
+    public void indexMapValuesQueryEntries() throws Throwable
+    {
+        createPopulatedMap("CREATE CUSTOM INDEX ON %s(VALUES(value)) USING 'StorageAttachedIndex'");
+        assertUnsupportedIndexOperator("SELECT * FROM %s WHERE value[1] = 'v1'");
+        assertEquals(2, execute("SELECT * FROM %s WHERE value[1] = 'v1' ALLOW FILTERING").size());
+    }
+
+    private void createPopulatedMap(String createIndex) throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int primary key, value map<int, text>)");
+        createIndex(createIndex);
+        waitForIndexQueryable();
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 1, new HashMap<Integer, String>() {{
+            put(1, "v1");
+            put(2, "v2");
+        }});
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 2, new HashMap<Integer, String>() {{
+            put(1, "v1");
+            put(2, "v3");
+        }});
+    }
+
+    private void createPopulatedFrozenMap(String createIndex) throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int primary key, value frozen<map<int, text>>)");
+        createIndex(createIndex);
+        waitForIndexQueryable();
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 1, new HashMap<Integer, String>() {{
+            put(1, "v1");
+            put(2, "v2");
+        }});
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 2, new HashMap<Integer, String>() {{
+            put(1, "v1");
+            put(2, "v3");
+        }});
+    }
+
+    private void createPopulatedFrozenList(String createIndex) throws Throwable
+    {
+        createTable("CREATE TABLE %s (pk int primary key, value frozen<list<int>>)");
+        createIndex(createIndex);
+        waitForIndexQueryable();
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 1, Arrays.asList(1, 2, 3));
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 2, Arrays.asList(1, 2, 3));
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 3, Arrays.asList(4, 5, 6));
+        execute("INSERT INTO %s (pk, value) VALUES (?, ?)", 4, Arrays.asList(1, 2, 7));
+    }
+
+    private void assertUnsupportedIndexOperator(String query, Object... values) throws Throwable
+    {
+        assertInvalidMessage(ALLOW_FILTERING_ERROR, query, values);
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/CompositePartitionKeyIndexTest.java b/test/unit/org/apache/cassandra/index/sai/cql/CompositePartitionKeyIndexTest.java
new file mode 100644
index 0000000000..6bf9891659
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/CompositePartitionKeyIndexTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.SAITester;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+public class CompositePartitionKeyIndexTest extends SAITester
+{
+    @Before
+    public void createTableAndIndex()
+    {
+        createTable("CREATE TABLE %s (pk1 int, pk2 text, val int, PRIMARY KEY((pk1, pk2)))");
+        createIndex("CREATE CUSTOM INDEX ON %s(pk1) USING 'StorageAttachedIndex'");
+        createIndex("CREATE CUSTOM INDEX ON %s(pk2) USING 'StorageAttachedIndex'");
+
+        disableCompaction();
+    }
+
+    private void insertData1() throws Throwable
+    {
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (1, '1', 1)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (2, '2', 2)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (3, '3', 3)");
+    }
+
+    private void insertData2() throws Throwable
+    {
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (4, '4', 4)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (5, '5', 5)");
+        execute("INSERT INTO %s (pk1, pk2, val) VALUES (6, '6', 6)");
+    }
+
+    @Test
+    public void queryFromMemtable() throws Throwable
+    {
+        insertData1();
+        insertData2();
+        runQueries();
+    }
+
+    private Object[] expectedRow(int index) {
+        return row(index, Integer.toString(index), index);
+    }
+
+    private void runQueries() throws Throwable
+    {
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 = 2"),
+                expectedRow(2));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 > 1"),
+                expectedRow(2),
+                expectedRow(3),
+                expectedRow(4),
+                expectedRow(5),
+                expectedRow(6));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 >= 3"),
+                expectedRow(3),
+                expectedRow(4),
+                expectedRow(5),
+                expectedRow(6));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 < 3"),
+                expectedRow(1),
+                expectedRow(2));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 <= 3"),
+                expectedRow(1),
+                expectedRow(2),
+                expectedRow(3));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk2 = '2'"),
+                expectedRow(2));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 > 1 AND pk2 = '2' ALLOW FILTERING"),
+                expectedRow(2));
+
+        assertRowsIgnoringOrder(execute("SELECT * FROM %s WHERE pk1 = -1 AND pk2 = '2' ALLOW FILTERING"));
+
+        assertThatThrownBy(()->execute("SELECT * FROM %s WHERE pk1 = -1 AND val = 2"))
+                .hasMessageContaining("use ALLOW FILTERING");
+
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/cql/DecimalLargeValueTest.java b/test/unit/org/apache/cassandra/index/sai/cql/DecimalLargeValueTest.java
new file mode 100644
index 0000000000..37ccda9421
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/sai/cql/DecimalLargeValueTest.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.cql;
+
+import java.math.BigDecimal;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.cassandra.index.sai.SAITester;
+
... 13122 lines suppressed ...


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


[cassandra] 03/06: Literal on-disk index and index write path (#9)

Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

maedhroz pushed a commit to branch cep-7-sai
in repository https://gitbox.apache.org/repos/asf/cassandra.git

commit 3914fb6674a8d1433d3208ab39ab0864061e9e94
Author: Mike Adamson <mi...@gmail.com>
AuthorDate: Thu Apr 13 17:23:13 2023 +0100

    Literal on-disk index and index write path (#9)
    
    This commit contains the following additions
     to SAI:
     - The index write path and index building
       based around StorageAttachedIndexBuilder
       and StorageAttachedIndexWriter
     - The on-disk index versioning using the
       SSTable Descriptor analog IndexDescriptor
       with Version and OnDiskFormat
     - The literal on-disk index using the
       LiteralIndexWriter
    
    patch by Mike Adamson; reviewed by Caleb Rackliffe and Andres de la Peña for CASSANDRA-18062
    
    Co-authored-by: Mike Adamson <mi...@gmail.com>
    Co-authored-by: Caleb Rackliffe <ca...@gmail.com>
    Co-authored-by: Andres de la Peña <a....@gmail.com>
    Co-authored-by: Piotr Kołaczkowski <pk...@gmail.com>
    Co-authored-by: Jason Rutherglen <ja...@gmail.com>
---
 conf/cassandra.yaml                                |   6 +
 .../config/CassandraRelevantProperties.java        |   3 +
 src/java/org/apache/cassandra/config/Config.java   |   2 +
 .../cassandra/config/DatabaseDescriptor.java       |   8 +-
 .../config/StorageAttachedIndexOptions.java        |  44 +
 .../statements/schema/CreateIndexStatement.java    |  80 +-
 src/java/org/apache/cassandra/db/ReadCommand.java  |   1 -
 .../org/apache/cassandra/db/lifecycle/Tracker.java |   6 +
 .../cassandra/db/memtable/AbstractMemtable.java    |  23 +
 .../org/apache/cassandra/db/memtable/Flushing.java |   7 +
 .../org/apache/cassandra/db/memtable/Memtable.java |   5 +
 .../apache/cassandra/index/sai/IndexContext.java   | 306 ++++---
 .../apache/cassandra/index/sai/QueryContext.java   |  11 +-
 .../apache/cassandra/index/sai/SSTableContext.java | 190 +++++
 .../cassandra/index/sai/SSTableContextManager.java | 148 ++++
 .../cassandra/index/sai/StorageAttachedIndex.java  | 241 +++++-
 .../index/sai/StorageAttachedIndexBuilder.java     | 352 ++++++++
 .../index/sai/StorageAttachedIndexGroup.java       | 214 ++++-
 .../index/sai/analyzer/AbstractAnalyzer.java       |   2 -
 .../index/sai/disk/IndexSearchResultIterator.java  | 141 ++++
 .../index/sai/disk/PerColumnIndexWriter.java       |  36 +-
 .../PerSSTableIndexWriter.java}                    |  25 +-
 .../cassandra/index/sai/disk/PrimaryKeyMap.java    |  76 ++
 .../cassandra/index/sai/disk/RowMapping.java       | 175 ++++
 .../cassandra/index/sai/disk/SSTableIndex.java     | 260 ++++++
 .../index/sai/disk/StorageAttachedIndexWriter.java | 258 ++++++
 .../index/sai/disk/format/IndexComponent.java      |  88 ++
 .../index/sai/disk/format/IndexDescriptor.java     | 433 ++++++++++
 .../index/sai/disk/format/OnDiskFormat.java        | 165 ++++
 .../cassandra/index/sai/disk/format/Version.java   | 154 ++++
 .../index/sai/disk/io/IndexFileUtils.java          | 148 ++++
 .../index/sai/disk/io/IndexInputReader.java        | 141 ++++
 .../index/sai/disk/io/IndexOutputWriter.java       | 121 +++
 .../index/sai/disk/io/RAMIndexOutput.java          |  86 ++
 .../sai/disk/io/SeekingRandomAccessInput.java      |  70 ++
 .../cassandra/index/sai/disk/v1/DirectReaders.java | 250 ++++++
 .../cassandra/index/sai/disk/v1/LongArray.java     |  90 ++
 .../index/sai/disk/v1/MemtableIndexWriter.java     | 179 ++++
 .../index/sai/disk/v1/MetadataSource.java          |  96 +++
 .../index/sai/disk/v1/MetadataWriter.java          |  85 ++
 .../index/sai/disk/v1/PerColumnIndexFiles.java     |  81 ++
 .../index/sai/disk/v1/RowAwarePrimaryKeyMap.java   | 198 +++++
 .../cassandra/index/sai/disk/v1/SAICodecUtils.java | 259 ++++++
 .../index/sai/disk/v1/SSTableComponentsWriter.java |  90 ++
 .../index/sai/disk/v1/SSTableIndexWriter.java      | 336 ++++++++
 .../index/sai/disk/v1/V1OnDiskFormat.java          | 247 ++++++
 .../index/sai/disk/v1/V1SSTableIndex.java          | 204 +++++
 .../disk/v1/bitpack/AbstractBlockPackedReader.java |  68 ++
 .../disk/v1/bitpack/AbstractBlockPackedWriter.java | 124 +++
 .../sai/disk/v1/bitpack/BlockPackedReader.java     | 128 +++
 .../sai/disk/v1/bitpack/BlockPackedWriter.java     |  77 ++
 .../v1/bitpack/MonotonicBlockPackedReader.java     | 120 +++
 .../v1/bitpack/MonotonicBlockPackedWriter.java     |  87 ++
 .../v1/bitpack/NumericValuesMeta.java}             |  29 +-
 .../sai/disk/v1/bitpack/NumericValuesWriter.java   | 100 +++
 .../sai/disk/v1/postings/MergePostingList.java     | 130 +++
 .../disk/v1/postings/PostingListRangeIterator.java | 171 ++++
 .../index/sai/disk/v1/postings/PostingsReader.java | 366 ++++++++
 .../index/sai/disk/v1/postings/PostingsWriter.java | 316 +++++++
 .../sai/disk/v1/segment/IndexSegmentSearcher.java  |  93 +++
 .../v1/segment/IndexSegmentSearcherContext.java    |  55 ++
 .../v1/segment/LiteralIndexSegmentSearcher.java    | 109 +++
 .../v1/segment/LiteralIndexSegmentTermsReader.java | 175 ++++
 .../index/sai/disk/v1/segment/Segment.java         | 129 +++
 .../index/sai/disk/v1/segment/SegmentBuilder.java  | 260 ++++++
 .../index/sai/disk/v1/segment/SegmentMetadata.java | 357 ++++++++
 .../sai/disk/v1/sortedterms/SortedTermsMeta.java   |  48 ++
 .../sai/disk/v1/sortedterms/SortedTermsReader.java | 234 ++++++
 .../sai/disk/v1/sortedterms/SortedTermsWriter.java | 214 +++++
 .../sai/disk/v1/sortedterms/package-info.java      |  58 ++
 .../index/sai/disk/v1/trie/LiteralIndexWriter.java | 111 +++
 .../index/sai/disk/v1/trie/TriePrefixSearcher.java |  98 +++
 .../disk/v1/trie/TrieTermsDictionaryReader.java    |  93 +++
 .../disk/v1/trie/TrieTermsDictionaryWriter.java    |  99 +++
 .../KeyRangeConcatIterator.java                    |   8 +-
 .../KeyRangeIntersectionIterator.java              |   6 +-
 .../sai/{utils => iterators}/KeyRangeIterator.java |   3 +-
 .../KeyRangeUnionIterator.java                     |   3 +-
 .../index/sai/memory/ByteSliceReader.java          | 124 +++
 .../index/sai/memory/InMemoryKeyRangeIterator.java |   2 +-
 .../cassandra/index/sai/memory/MemtableIndex.java  |   2 +-
 .../index/sai/memory/MemtableIndexManager.java     | 157 ++++
 .../index/sai/memory/MemtableTermsIterator.java    | 135 +++
 .../index/sai/memory/RAMPostingSlices.java         | 137 +++
 .../index/sai/memory/RAMStringIndexer.java         | 167 ++++
 .../index/sai/memory/TrieMemoryIndex.java          |   2 +-
 .../index/sai/metrics/ColumnQueryMetrics.java      |  91 ++
 ...bleStateMetrics.java => IndexGroupMetrics.java} |  16 +-
 .../cassandra/index/sai/metrics/IndexMetrics.java  |  35 +-
 .../sai/metrics/MulticastQueryEventListeners.java  |  90 ++
 .../index/sai/metrics/QueryEventListener.java      |  75 ++
 .../index/sai/metrics/TableQueryMetrics.java       |  55 +-
 .../index/sai/metrics/TableStateMetrics.java       |  17 +-
 .../cassandra/index/sai/plan/Expression.java       |   5 +
 .../apache/cassandra/index/sai/plan/Operation.java |   2 +-
 .../cassandra/index/sai/plan/QueryController.java  |  41 +-
 .../cassandra/index/sai/plan/QueryViewBuilder.java | 192 +++++
 .../sai/plan/StorageAttachedIndexQueryPlan.java    |   4 +-
 .../sai/plan/StorageAttachedIndexSearcher.java     |  11 +-
 .../index/sai/postings/OrdinalPostingList.java     |  18 +-
 .../index/sai/postings/PeekablePostingList.java    | 116 +++
 .../cassandra/index/sai/postings/PostingList.java  |  60 ++
 .../index/sai/utils/NamedMemoryLimiter.java        |  79 ++
 .../cassandra/index/sai/utils/PrimaryKey.java      | 348 +++++---
 .../index/sai/utils/PrimaryKeyFactory.java         |  57 --
 .../cassandra/index/sai/utils/TermsIterator.java   |  46 +
 .../apache/cassandra/index/sai/utils/TypeUtil.java |   9 +
 .../cassandra/index/sai/view/IndexViewManager.java | 150 ++++
 .../cassandra/index/sai/view/RangeTermTree.java    | 134 +++
 .../org/apache/cassandra/index/sai/view/View.java  | 100 +++
 ...ystemView.java => ColumnIndexesSystemView.java} |   6 +-
 .../sai/virtual/SSTableIndexesSystemView.java      | 132 +++
 .../index/sai/virtual/SegmentsSystemView.java      | 125 +++
 .../sai/virtual/StorageAttachedIndexTables.java    |   6 +-
 .../io/compress/CompressedSequentialWriter.java    |  35 +
 .../apache/cassandra/io/sstable/Descriptor.java    |  10 +-
 .../org/apache/cassandra/io/sstable/SSTable.java   |  42 +-
 .../cassandra/io/sstable/format/TOCComponent.java  |  11 +
 .../apache/cassandra/io/tries/ValueIterator.java   |  38 +-
 src/java/org/apache/cassandra/io/tries/Walker.java |  76 ++
 .../apache/cassandra/io/util/DataOutputPlus.java   |   9 +-
 .../org/apache/cassandra/io/util/PageAware.java    |   2 +-
 .../io/util/TailOverridingRebufferer.java          |   2 +-
 .../utils/bytecomparable/ByteComparable.java       |   5 +
 .../utils/concurrent/LightweightRecycler.java      |   2 +-
 .../cassandra/utils/concurrent/ThreadLocals.java   |   2 +-
 .../cassandra/distributed/test/sai/SAIUtil.java    |   4 +-
 .../config/DatabaseDescriptorRefTest.java          |   8 +-
 .../config/StorageAttachedIndexOptionsTest.java    |  44 +
 .../unit/org/apache/cassandra/index/StubIndex.java |   3 +-
 .../cassandra/index/sai/IndexingSchemaLoader.java  | 173 ----
 .../org/apache/cassandra/index/sai/SAITester.java  | 650 ++++++++++++++-
 .../index/sai/cql/AbstractQueryTester.java         |   4 +-
 .../cassandra/index/sai/cql/BaseDataModel.java     |  84 +-
 .../index/sai/cql/CollectionIndexingTest.java      |   2 +-
 .../index/sai/cql/DecimalLargeValueTest.java       |   6 +-
 .../cassandra/index/sai/cql/IndexQuerySupport.java |  52 +-
 .../sai/cql/MixedIndexImplementationsTest.java     |   2 +
 .../index/sai/cql/StorageAttachedIndexDDLTest.java | 924 +++++++++++++++++++--
 .../cassandra/index/sai/cql/types/AsciiTest.java   |   2 +-
 .../cassandra/index/sai/cql/types/BigintTest.java  |   2 +-
 .../cassandra/index/sai/cql/types/BooleanTest.java |   2 +-
 .../cassandra/index/sai/cql/types/DataSet.java     |  20 +-
 .../cassandra/index/sai/cql/types/DateTest.java    |   2 +-
 .../cassandra/index/sai/cql/types/DecimalTest.java |   2 +-
 .../cassandra/index/sai/cql/types/DoubleTest.java  |   2 +-
 .../cassandra/index/sai/cql/types/FloatTest.java   |   2 +-
 .../index/sai/cql/types/IndexingTypeSupport.java   |  51 +-
 .../cassandra/index/sai/cql/types/InetTest.java    |   2 +-
 .../cassandra/index/sai/cql/types/IntTest.java     |   2 +-
 .../sai/cql/types/NumericTypeSortingTest.java      |  20 +-
 .../cassandra/index/sai/cql/types/QuerySet.java    | 126 ++-
 .../index/sai/cql/types/SmallintTest.java          |   2 +-
 .../cassandra/index/sai/cql/types/TextTest.java    |   2 +-
 .../cassandra/index/sai/cql/types/TimeTest.java    |   2 +-
 .../index/sai/cql/types/TimestampTest.java         |   2 +-
 .../index/sai/cql/types/TimeuuidTest.java          |   2 +-
 .../cassandra/index/sai/cql/types/TinyintTest.java |   2 +-
 .../cassandra/index/sai/cql/types/UuidTest.java    |   2 +-
 .../cassandra/index/sai/cql/types/VarintTest.java  |   2 +-
 .../cql/types/collections/CollectionDataSet.java   |  39 +-
 .../collections/lists/FrozenListAsciiTest.java     |   2 +-
 .../collections/lists/FrozenListDecimalTest.java   |   2 +-
 .../types/collections/lists/FrozenListIntTest.java |   2 +-
 .../collections/lists/FrozenListVarintTest.java    |   2 +-
 .../cql/types/collections/lists/ListAsciiTest.java |   2 +-
 .../types/collections/lists/ListBigintTest.java    |   2 +-
 .../cql/types/collections/lists/ListDateTest.java  |   2 +-
 .../types/collections/lists/ListDecimalTest.java   |   2 +-
 .../types/collections/lists/ListDoubleTest.java    |   2 +-
 .../cql/types/collections/lists/ListFloatTest.java |   2 +-
 .../lists/ListFrozenCollectionTest.java            |   2 +-
 .../cql/types/collections/lists/ListInetTest.java  |   2 +-
 .../cql/types/collections/lists/ListIntTest.java   |   2 +-
 .../types/collections/lists/ListSmallintTest.java  |   2 +-
 .../cql/types/collections/lists/ListTextTest.java  |   2 +-
 .../cql/types/collections/lists/ListTimeTest.java  |   2 +-
 .../types/collections/lists/ListTimestampTest.java |   2 +-
 .../types/collections/lists/ListTimeuuidTest.java  |   2 +-
 .../types/collections/lists/ListTinyintTest.java   |   2 +-
 .../cql/types/collections/lists/ListUuidTest.java  |   2 +-
 .../types/collections/lists/ListVarintTest.java    |   2 +-
 .../types/collections/maps/FrozenMapAsciiTest.java |   2 +-
 .../collections/maps/FrozenMapDecimalTest.java     |   2 +-
 .../types/collections/maps/FrozenMapIntTest.java   |   2 +-
 .../collections/maps/FrozenMapVarintTest.java      |   2 +-
 .../cql/types/collections/maps/MapAsciiTest.java   |   2 +-
 .../cql/types/collections/maps/MapBigintTest.java  |   2 +-
 .../cql/types/collections/maps/MapDateTest.java    |   2 +-
 .../cql/types/collections/maps/MapDecimalTest.java |   2 +-
 .../cql/types/collections/maps/MapDoubleTest.java  |   2 +-
 .../collections/maps/MapEntriesAsciiTest.java      |   2 +-
 .../maps/MapEntriesFrozenCollectionTest.java       |   2 +-
 .../types/collections/maps/MapEntriesIntTest.java  |   2 +-
 .../collections/maps/MapEntriesVarintTest.java     |   2 +-
 .../cql/types/collections/maps/MapFloatTest.java   |   2 +-
 .../collections/maps/MapFrozenCollectionTest.java  |   2 +-
 .../cql/types/collections/maps/MapInetTest.java    |   2 +-
 .../sai/cql/types/collections/maps/MapIntTest.java |   2 +-
 .../types/collections/maps/MapKeysAsciiTest.java   |   2 +-
 .../maps/MapKeysFrozenCollectionTest.java          |   2 +-
 .../cql/types/collections/maps/MapKeysIntTest.java |   2 +-
 .../types/collections/maps/MapKeysVarintTest.java  |   2 +-
 .../types/collections/maps/MapSmallintTest.java    |   2 +-
 .../cql/types/collections/maps/MapTextTest.java    |   2 +-
 .../cql/types/collections/maps/MapTimeTest.java    |   2 +-
 .../types/collections/maps/MapTimestampTest.java   |   2 +-
 .../types/collections/maps/MapTimeuuidTest.java    |   2 +-
 .../cql/types/collections/maps/MapTinyintTest.java |   2 +-
 .../cql/types/collections/maps/MapUuidTest.java    |   2 +-
 .../types/collections/maps/MapValuesAsciiTest.java |   2 +-
 .../maps/MapValuesFrozenCollectionTest.java        |   2 +-
 .../types/collections/maps/MapValuesIntTest.java   |   2 +-
 .../collections/maps/MapValuesVarintTest.java      |   2 +-
 .../cql/types/collections/maps/MapVarintTest.java  |   2 +-
 .../types/collections/maps/MultiMapAsciiTest.java  |   2 +-
 .../types/collections/maps/MultiMapIntTest.java    |   2 +-
 .../types/collections/maps/MultiMapVarintTest.java |   2 +-
 .../types/collections/sets/FrozenSetAsciiTest.java |   2 +-
 .../collections/sets/FrozenSetDecimalTest.java     |   2 +-
 .../types/collections/sets/FrozenSetIntTest.java   |   2 +-
 .../collections/sets/FrozenSetVarintTest.java      |   2 +-
 .../cql/types/collections/sets/SetAsciiTest.java   |   2 +-
 .../cql/types/collections/sets/SetBigintTest.java  |   2 +-
 .../cql/types/collections/sets/SetDateTest.java    |   2 +-
 .../cql/types/collections/sets/SetDecimalTest.java |   2 +-
 .../cql/types/collections/sets/SetDoubleTest.java  |   2 +-
 .../cql/types/collections/sets/SetFloatTest.java   |   2 +-
 .../collections/sets/SetFrozenCollectionTest.java  |   2 +-
 .../cql/types/collections/sets/SetInetTest.java    |   2 +-
 .../sai/cql/types/collections/sets/SetIntTest.java |   2 +-
 .../types/collections/sets/SetSmallintTest.java    |   2 +-
 .../cql/types/collections/sets/SetTextTest.java    |   2 +-
 .../cql/types/collections/sets/SetTimeTest.java    |   2 +-
 .../types/collections/sets/SetTimestampTest.java   |   2 +-
 .../types/collections/sets/SetTimeuuidTest.java    |   2 +-
 .../cql/types/collections/sets/SetTinyintTest.java |   2 +-
 .../cql/types/collections/sets/SetUuidTest.java    |   2 +-
 .../cql/types/collections/sets/SetVarintTest.java  |   2 +-
 .../types/multicell/FrozenTupleCollectionTest.java |   2 +-
 .../cql/types/multicell/FrozenTupleDataSet.java    |   2 +-
 .../sai/cql/types/multicell/FrozenTupleTest.java   |   2 +-
 .../cql/types/multicell/FrozenTupleTupleTest.java  |   2 +-
 .../types/multicell/FrozenUDTCollectionTest.java   |   2 +-
 .../sai/cql/types/multicell/FrozenUDTDataSet.java  |   6 +-
 .../sai/cql/types/multicell/FrozenUDTTest.java     |   2 +-
 .../index/sai/cql/types/multicell/TupleTest.java   |   2 +-
 .../cassandra/index/sai/disk/ArrayPostingList.java |  92 ++
 .../cassandra/index/sai/disk/NodeStartupTest.java  | 407 +++++++++
 .../index/sai/disk/SelectiveIntersectionTest.java  | 159 ++++
 .../index/sai/disk/SingleNodeQueryFailureTest.java | 111 +++
 .../index/sai/disk/format/IndexDescriptorTest.java | 122 +++
 .../index/sai/disk/format/VersionTest.java         |  50 ++
 .../index/sai/disk/io/TrackingIndexFileUtils.java  | 116 +++
 .../index/sai/disk/v1/InvertedIndexBuilder.java    |  61 ++
 .../sai/disk/v1/InvertedIndexSearcherTest.java     | 218 +++++
 .../cassandra/index/sai/disk/v1/MetadataTest.java  | 189 +++++
 .../index/sai/disk/v1/SAICodecUtilsTest.java       | 390 +++++++++
 .../index/sai/disk/v1/SegmentFlushTest.java        | 222 +++++
 .../cassandra/index/sai/disk/v1/SegmentTest.java   | 165 ++++
 .../cassandra/index/sai/disk/v1/SorterTest.java    |  71 ++
 .../index/sai/disk/v1/TermsReaderTest.java         | 180 ++++
 .../cassandra/index/sai/disk/v1/TermsScanner.java  |  96 +++
 .../sai/disk/v1/bitpack/NumericValuesTest.java     | 125 +++
 .../sai/disk/v1/postings/MergePostingListTest.java | 333 ++++++++
 .../index/sai/disk/v1/postings/PostingsTest.java   | 394 +++++++++
 .../disk/v1/postings/ScanningPostingsReader.java   |  45 +
 .../sai/disk/v1/sortedterms/SortedTermsTest.java   | 415 +++++++++
 .../sai/disk/v1/trie/TriePrefixSearcherTest.java   | 160 ++++
 .../sai/disk/v1/trie/TrieTermsDictionaryTest.java  | 171 ++++
 .../index/sai/disk/v1/trie/TrieTermsIterator.java  |  82 ++
 .../index/sai/functional/CompactionTest.java       | 339 ++++++++
 .../index/sai/functional/DiskSpaceTest.java        |  56 ++
 .../index/sai/functional/DropTableTest.java        | 106 +++
 .../index/sai/functional/FailureTest.java          | 125 +++
 .../index/sai/functional/FlushingTest.java         |  60 ++
 .../index/sai/functional/GroupComponentsTest.java  | 120 +++
 .../index/sai/functional/NodeRestartTest.java      | 180 ++++
 .../index/sai/functional/SnapshotTest.java         | 167 ++++
 .../AbstractKeyRangeIteratorTester.java            |   9 +-
 .../KeyRangeConcatIteratorTest.java                |  11 +-
 .../KeyRangeIntersectionIteratorTest.java          |   4 +-
 .../KeyRangeUnionIteratorTest.java                 |  76 +-
 .../sai/{utils => iterators}/LongIterator.java     |   7 +-
 .../AbstractInMemoryKeyRangeIteratorTester.java    |   7 +-
 .../sai/memory/InMemoryKeyRangeIteratorTest.java   |   2 +-
 .../PriorityInMemoryKeyRangeIteratorTest.java      |   2 +-
 .../index/sai/memory/RAMPostingSlicesTest.java     |  80 ++
 .../index/sai/memory/RAMStringIndexerTest.java     | 115 +++
 .../index/sai/memory/TrieMemoryIndexTest.java      |   2 +-
 .../FinalSegmentFlushingFailureTest.java}          |  14 +-
 .../index/sai/metrics/IndexGroupMetricsTest.java   |  92 ++
 .../index/sai/metrics/IndexMetricsTest.java        |   8 +-
 .../index/sai/metrics/QueryMetricsTest.java        |  16 +-
 .../sai/metrics/SegmentFlushingFailureTester.java  | 217 +++++
 .../index/sai/metrics/StateMetricsTest.java        |  14 +-
 .../TinySegmentFlushingFailureTest.java}           |  20 +-
 .../cassandra/index/sai/plan/OperationTest.java    | 179 ++--
 .../sai/postings/PeekablePostingListTest.java      |  73 ++
 .../index/sai/utils/IndexInputLeakDetector.java    |  62 ++
 .../index/sai/utils/NamedMemoryLimiterTest.java    |  82 ++
 .../cassandra/index/sai/utils/PrimaryKeyTest.java  |  28 +-
 .../index/sai/utils/SAIRandomizedTester.java       |  89 +-
 .../cassandra/index/sai/utils/TypeUtilTest.java    | 251 ++++++
 .../index/sai/virtual/IndexesSystemViewTest.java   |  70 +-
 .../index/sai/virtual/SSTablesSystemViewTest.java  | 201 +++++
 .../index/sai/virtual/SegmentsSystemViewTest.java  | 220 +++++
 .../cassandra/io/tries/AbstractTrieTestBase.java   |   6 +
 .../org/apache/cassandra/io/tries/WalkerTest.java  |   2 +-
 309 files changed, 21596 insertions(+), 1302 deletions(-)

diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index ce39c019ee..4c8f868a8b 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -1511,6 +1511,12 @@ transparent_data_encryption_options:
           store_type: JCEKS
           key_password: cassandra
 
+# Storage Attached Indexing options.
+# sai_options:
+  ## Total permitted memory allowed for writing SAI index segments. This memory
+  ## is split between all SAI indexes being built so more indexes will mean smaller
+  ## segment sizes.
+  # segment_write_buffer_size: 1024MiB
 
 #####################
 # SAFETY THRESHOLDS #
diff --git a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
index c3fdb8aed2..99dc65cdb6 100644
--- a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
+++ b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
@@ -395,6 +395,9 @@ public enum CassandraRelevantProperties
     SEARCH_CONCURRENCY_FACTOR("cassandra.search_concurrency_factor", "1"),
 
     // SAI specific properties
+    /** Latest version to be used for SAI index writing */
+    SAI_LATEST_VERSION("cassandra.sai.latest_version", "aa"),
+
     /** Controls the maximum number of index query intersections that will take part in a query */
     SAI_INTERSECTION_CLAUSE_LIMIT("cassandra.sai.intersection.clause.limit", "2"),
 
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index 56ecb199d4..14c0b00f5a 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -781,6 +781,8 @@ public class Config
      */
     public volatile double range_tombstone_list_growth_factor = 1.5;
 
+    public StorageAttachedIndexOptions sai_options = new StorageAttachedIndexOptions();
+
     /**
      * @deprecated migrate to {@link DatabaseDescriptor#isClientInitialized()}
      */
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 424ac67cf0..84d85e8e46 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -37,7 +37,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
-import java.util.OptionalDouble;
 import java.util.ServiceLoader;
 import java.util.Set;
 import java.util.UUID;
@@ -966,6 +965,8 @@ public class DatabaseDescriptor
 
         if (conf.dump_heap_on_uncaught_exception && DatabaseDescriptor.getHeapDumpPath() == null)
             throw new ConfigurationException(String.format("Invalid configuration. Heap dump is enabled but cannot create heap dump output path: %s.", conf.heap_dump_path != null ? conf.heap_dump_path : "null"));
+
+        conf.sai_options.validate();
     }
 
     @VisibleForTesting
@@ -4775,4 +4776,9 @@ public class DatabaseDescriptor
                OptionalDouble.of(conf.severity_during_decommission) :
                OptionalDouble.empty();
     }
+
+    public static DataStorageSpec.IntMebibytesBound getSAISegmentWriteBufferSpace()
+    {
+        return conf.sai_options.segment_write_buffer_size;
+    }
 }
diff --git a/src/java/org/apache/cassandra/config/StorageAttachedIndexOptions.java b/src/java/org/apache/cassandra/config/StorageAttachedIndexOptions.java
new file mode 100644
index 0000000000..18bf3506fe
--- /dev/null
+++ b/src/java/org/apache/cassandra/config/StorageAttachedIndexOptions.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.config;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.exceptions.ConfigurationException;
+
+public class StorageAttachedIndexOptions
+{
+    public static final int DEFAULT_SEGMENT_BUFFER_MB = 1024;
+
+    @VisibleForTesting
+    public static final int MAXIMUM_SEGMENT_BUFFER_MB = 32768;
+
+    @VisibleForTesting
+    public static final String INVALID_BUFFER_SIZE_ERROR = "Invalid value for segment_write_buffer_size. " +
+                                                           "Value must be a positive integer less than " + MAXIMUM_SEGMENT_BUFFER_MB + "MiB";
+
+    public DataStorageSpec.IntMebibytesBound segment_write_buffer_size = new DataStorageSpec.IntMebibytesBound(DEFAULT_SEGMENT_BUFFER_MB);
+
+    public void validate()
+    {
+        if (segment_write_buffer_size.toMebibytes() > MAXIMUM_SEGMENT_BUFFER_MB)
+        {
+            throw new ConfigurationException(INVALID_BUFFER_SIZE_ERROR);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java
index 73a7b6aac6..73b66b738d 100644
--- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java
+++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateIndexStatement.java
@@ -48,8 +48,35 @@ import static com.google.common.collect.Iterables.tryFind;
 
 public final class CreateIndexStatement extends AlterSchemaStatement
 {
+    public static final String SASI_INDEX_DISABLED = "SASI indexes are disabled. Enable in cassandra.yaml to use.";
+    public static final String KEYSPACE_DOES_NOT_EXIST = "Keyspace '%s' doesn't exist";
+    public static final String TABLE_DOES_NOT_EXIST = "Table '%s' doesn't exist";
+    public static final String COUNTER_TABLES_NOT_SUPPORTED = "Secondary indexes on counter tables aren't supported";
+    public static final String MATERIALIZED_VIEWS_NOT_SUPPORTED = "Secondary indexes on materialized views aren't supported";
+    public static final String TRANSIENTLY_REPLICATED_KEYSPACE_NOT_SUPPORTED = "Secondary indexes are not supported on transiently replicated keyspaces";
+    public static final String CUSTOM_CREATE_WITHOUT_COLUMN = "Only CUSTOM indexes can be created without specifying a target column";
+    public static final String CUSTOM_MULTIPLE_COLUMNS = "Only CUSTOM indexes support multiple columns";
+    public static final String DUPLICATE_TARGET_COLUMN = "Duplicate column '%s' in index target list";
+    public static final String COLUMN_DOES_NOT_EXIST = "Column '%s' doesn't exist";
     public static final String INVALID_CUSTOM_INDEX_TARGET = "Column '%s' is longer than the permissible name length of %d characters or" +
                                                              " contains non-alphanumeric-underscore characters";
+    public static final String COLLECTIONS_WITH_DURATIONS_NOT_SUPPORTED = "Secondary indexes are not supported on collections containing durations";
+    public static final String TUPLES_WITH_DURATIONS_NOT_SUPPORTED = "Secondary indexes are not supported on tuples containing durations";
+    public static final String DURATIONS_NOT_SUPPORTED = "Secondary indexes are not supported on duration columns";
+    public static final String UDTS_WITH_DURATIONS_NOT_SUPPORTED = "Secondary indexes are not supported on UDTs containing durations";
+    public static final String PRIMARY_KEY_IN_COMPACT_STORAGE = "Secondary indexes are not supported on PRIMARY KEY columns in COMPACT STORAGE tables";
+    public static final String COMPACT_COLUMN_IN_COMPACT_STORAGE = "Secondary indexes are not supported on compact value column of COMPACT STORAGE tables";
+    public static final String ONLY_PARTITION_KEY = "Cannot create secondary index on the only partition key column %s";
+    public static final String CREATE_ON_FROZEN_COLUMN = "Cannot create %s() index on frozen column %s. Frozen collections are immutable and must be fully " +
+                                                         "indexed by using the 'full(%s)' modifier";
+    public static final String FULL_ON_FROZEN_COLLECTIONS = "full() indexes can only be created on frozen collections";
+    public static final String NON_COLLECTION_SIMPLE_INDEX = "Cannot create %s() index on %s. Non-collection columns only support simple indexes";
+    public static final String CREATE_WITH_NON_MAP_TYPE = "Cannot create index on %s of column %s with non-map type";
+    public static final String CREATE_ON_NON_FROZEN_UDT = "Cannot create index on non-frozen UDT column %s";
+    public static final String INDEX_ALREADY_EXISTS = "Index '%s' already exists";
+    public static final String INDEX_DUPLICATE_OF_EXISTING = "Index %s is a duplicate of existing index %s";
+    public static final String KEYSPACE_DOES_NOT_MATCH_TABLE = "Keyspace name '%s' doesn't match table name '%s'";
+    public static final String KEYSPACE_DOES_NOT_MATCH_INDEX = "Keyspace name '%s' doesn't match index name '%s'";
 
     private final String indexName;
     private final String tableName;
@@ -90,32 +117,32 @@ public final class CreateIndexStatement extends AlterSchemaStatement
         Guardrails.createSecondaryIndexesEnabled.ensureEnabled("Creating secondary indexes", state);
 
         if (attrs.isCustom && attrs.customClass.equals(SASIIndex.class.getName()) && !DatabaseDescriptor.getSASIIndexesEnabled())
-            throw new InvalidRequestException("SASI indexes are disabled. Enable in cassandra.yaml to use.");
+            throw new InvalidRequestException(SASI_INDEX_DISABLED);
 
         KeyspaceMetadata keyspace = schema.getNullable(keyspaceName);
         if (null == keyspace)
-            throw ire("Keyspace '%s' doesn't exist", keyspaceName);
+            throw ire(KEYSPACE_DOES_NOT_EXIST, keyspaceName);
 
         TableMetadata table = keyspace.getTableOrViewNullable(tableName);
         if (null == table)
-            throw ire("Table '%s' doesn't exist", tableName);
+            throw ire(TABLE_DOES_NOT_EXIST, tableName);
 
         if (null != indexName && keyspace.hasIndex(indexName))
         {
             if (ifNotExists)
                 return schema;
 
-            throw ire("Index '%s' already exists", indexName);
+            throw ire(INDEX_ALREADY_EXISTS, indexName);
         }
 
         if (table.isCounter())
-            throw ire("Secondary indexes on counter tables aren't supported");
+            throw ire(COUNTER_TABLES_NOT_SUPPORTED);
 
         if (table.isView())
-            throw ire("Secondary indexes on materialized views aren't supported");
+            throw ire(MATERIALIZED_VIEWS_NOT_SUPPORTED);
 
         if (Keyspace.open(table.keyspace).getReplicationStrategy().hasTransientReplicas())
-            throw new InvalidRequestException("Secondary indexes are not supported on transiently replicated keyspaces");
+            throw new InvalidRequestException(TRANSIENTLY_REPLICATED_KEYSPACE_NOT_SUPPORTED);
 
         // guardrails to limit number of secondary indexes per table.
         Guardrails.secondaryIndexesPerTable.guard(table.indexes.size() + 1,
@@ -128,17 +155,17 @@ public final class CreateIndexStatement extends AlterSchemaStatement
         List<IndexTarget> indexTargets = Lists.newArrayList(transform(rawIndexTargets, t -> t.prepare(table)));
 
         if (indexTargets.isEmpty() && !attrs.isCustom)
-            throw ire("Only CUSTOM indexes can be created without specifying a target column");
+            throw ire(CUSTOM_CREATE_WITHOUT_COLUMN);
 
         if (indexTargets.size() > 1)
         {
             if (!attrs.isCustom)
-                throw ire("Only CUSTOM indexes support multiple columns");
+                throw ire(CUSTOM_MULTIPLE_COLUMNS);
 
             Set<ColumnIdentifier> columns = new HashSet<>();
             for (IndexTarget target : indexTargets)
                 if (!columns.add(target.column))
-                    throw ire("Duplicate column '%s' in index target list", target.column);
+                    throw ire(DUPLICATE_TARGET_COLUMN, target.column);
         }
 
         IndexMetadata.Kind kind = attrs.isCustom ? IndexMetadata.Kind.CUSTOM : IndexMetadata.Kind.COMPOSITES;
@@ -158,7 +185,7 @@ public final class CreateIndexStatement extends AlterSchemaStatement
             if (ifNotExists)
                 return schema;
 
-            throw ire("Index %s is a duplicate of existing index %s", index.name, equalIndex.name);
+            throw ire(INDEX_DUPLICATE_OF_EXISTING, index.name, equalIndex.name);
         }
 
         TableMetadata newTable = table.withSwapped(table.indexes.with(index));
@@ -181,7 +208,7 @@ public final class CreateIndexStatement extends AlterSchemaStatement
         ColumnMetadata column = table.getColumn(target.column);
 
         if (null == column)
-            throw ire("Column '%s' doesn't exist", target.column);
+            throw ire(COLUMN_DOES_NOT_EXIST, target.column);
 
         if ((kind == IndexMetadata.Kind.CUSTOM) && !SchemaConstants.isValidName(target.column.toString()))
             throw ire(INVALID_CUSTOM_INDEX_TARGET, target.column, SchemaConstants.NAME_LENGTH);
@@ -189,44 +216,43 @@ public final class CreateIndexStatement extends AlterSchemaStatement
         if (column.type.referencesDuration())
         {
             if (column.type.isCollection())
-                throw ire("Secondary indexes are not supported on collections containing durations");
+                throw ire(COLLECTIONS_WITH_DURATIONS_NOT_SUPPORTED);
 
             if (column.type.isTuple())
-                throw ire("Secondary indexes are not supported on tuples containing durations");
+                throw ire(TUPLES_WITH_DURATIONS_NOT_SUPPORTED);
 
             if (column.type.isUDT())
-                throw  ire("Secondary indexes are not supported on UDTs containing durations");
+                throw  ire(UDTS_WITH_DURATIONS_NOT_SUPPORTED);
 
-            throw ire("Secondary indexes are not supported on duration columns");
+            throw ire(DURATIONS_NOT_SUPPORTED);
         }
 
         if (table.isCompactTable())
         {
             TableMetadata.CompactTableMetadata compactTable = (TableMetadata.CompactTableMetadata) table;
             if (column.isPrimaryKeyColumn())
-                throw new InvalidRequestException("Secondary indexes are not supported on PRIMARY KEY columns in COMPACT STORAGE tables");
+                throw new InvalidRequestException(PRIMARY_KEY_IN_COMPACT_STORAGE);
             if (compactTable.compactValueColumn.equals(column))
-                throw new InvalidRequestException("Secondary indexes are not supported on compact value column of COMPACT STORAGE tables");
+                throw new InvalidRequestException(COMPACT_COLUMN_IN_COMPACT_STORAGE);
         }
 
         if (column.isPartitionKey() && table.partitionKeyColumns().size() == 1)
-            throw ire("Cannot create secondary index on the only partition key column %s", column);
+            throw ire(ONLY_PARTITION_KEY, column);
 
         if (column.type.isFrozenCollection() && target.type != Type.FULL)
-            throw ire("Cannot create %s() index on frozen column %s. Frozen collections are immutable and must be fully " +
-                      "indexed by using the 'full(%s)' modifier", target.type, column, column);
+            throw ire(CREATE_ON_FROZEN_COLUMN, target.type, column, column);
 
         if (!column.type.isFrozenCollection() && target.type == Type.FULL)
-            throw ire("full() indexes can only be created on frozen collections");
+            throw ire(FULL_ON_FROZEN_COLLECTIONS);
 
         if (!column.type.isCollection() && target.type != Type.SIMPLE)
-            throw ire("Cannot create %s() index on %s. Non-collection columns only support simple indexes", target.type, column);
+            throw ire(NON_COLLECTION_SIMPLE_INDEX, target.type, column);
 
         if (!(column.type instanceof MapType && column.type.isMultiCell()) && (target.type == Type.KEYS || target.type == Type.KEYS_AND_VALUES))
-            throw ire("Cannot create index on %s of column %s with non-map type", target.type, column);
+            throw ire(CREATE_WITH_NON_MAP_TYPE, target.type, column);
 
         if (column.type.isUDT() && column.type.isMultiCell())
-            throw ire("Cannot create index on non-frozen UDT column %s", column);
+            throw ire(CREATE_ON_NON_FROZEN_UDT, column);
     }
 
     private String generateIndexName(KeyspaceMetadata keyspace, List<IndexTarget> targets)
@@ -286,10 +312,10 @@ public final class CreateIndexStatement extends AlterSchemaStatement
                                 : indexName.hasKeyspace() ? indexName.getKeyspace() : state.getKeyspace();
 
             if (tableName.hasKeyspace() && !keyspaceName.equals(tableName.getKeyspace()))
-                throw ire("Keyspace name '%s' doesn't match table name '%s'", keyspaceName, tableName);
+                throw ire(KEYSPACE_DOES_NOT_MATCH_TABLE, keyspaceName, tableName);
 
             if (indexName.hasKeyspace() && !keyspaceName.equals(indexName.getKeyspace()))
-                throw ire("Keyspace name '%s' doesn't match index name '%s'", keyspaceName, tableName);
+                throw ire(KEYSPACE_DOES_NOT_MATCH_INDEX, keyspaceName, tableName);
 
             return new CreateIndexStatement(keyspaceName, tableName.getName(), indexName.getName(), rawIndexTargets, attrs, ifNotExists);
         }
diff --git a/src/java/org/apache/cassandra/db/ReadCommand.java b/src/java/org/apache/cassandra/db/ReadCommand.java
index abbf099704..ba7a0c7570 100644
--- a/src/java/org/apache/cassandra/db/ReadCommand.java
+++ b/src/java/org/apache/cassandra/db/ReadCommand.java
@@ -408,7 +408,6 @@ public abstract class ReadCommand extends AbstractReadQuery
             {
                 cfs.indexManager.checkQueryability(indexQueryPlan);
 
-                Index index = indexQueryPlan.getFirst();
                 searcher = indexQueryPlan.searcherFor(this);
                 Tracing.trace("Executing read on {}.{} using index{} {}",
                               cfs.metadata.keyspace,
diff --git a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
index c54ee83bd7..575af6cff5 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
@@ -217,6 +217,12 @@ public class Tracker
         return accumulate;
     }
 
+    public void updateLiveDiskSpaceUsed(long adjustment)
+    {
+        cfstore.metric.liveDiskSpaceUsed.inc(adjustment);
+        cfstore.metric.totalDiskSpaceUsed.inc(adjustment);
+    }
+
     // SETUP / CLEANUP
 
     public void addInitialSSTables(Iterable<SSTableReader> sstables)
diff --git a/src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java b/src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java
index 8e50456d6b..dee47a1928 100644
--- a/src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java
+++ b/src/java/org/apache/cassandra/db/memtable/AbstractMemtable.java
@@ -30,6 +30,7 @@ import com.google.common.annotations.VisibleForTesting;
 
 import org.apache.cassandra.db.RegularAndStaticColumns;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.partitions.Partition;
 import org.apache.cassandra.db.rows.EncodingStats;
 import org.apache.cassandra.schema.ColumnMetadata;
@@ -39,6 +40,7 @@ import org.github.jamm.Unmetered;
 
 public abstract class AbstractMemtable implements Memtable
 {
+    private final AtomicReference<LifecycleTransaction> flushTransaction = new AtomicReference<>(null);
     protected final AtomicLong currentOperations = new AtomicLong(0);
     protected final ColumnsCollector columnsCollector;
     protected final StatsCollector statsCollector = new StatsCollector();
@@ -66,21 +68,25 @@ public abstract class AbstractMemtable implements Memtable
         this.minTimestamp = new AtomicLong(minTimestamp);
     }
 
+    @Override
     public TableMetadata metadata()
     {
         return metadata.get();
     }
 
+    @Override
     public long operationCount()
     {
         return currentOperations.get();
     }
 
+    @Override
     public long getMinTimestamp()
     {
         return minTimestamp.get() != EncodingStats.NO_STATS.minTimestamp ? minTimestamp.get() : NO_MIN_TIMESTAMP;
     }
 
+    @Override
     public int getMinLocalDeletionTime()
     {
         return minLocalDeletionTime.get();
@@ -120,6 +126,18 @@ public abstract class AbstractMemtable implements Memtable
         return statsCollector.get();
     }
 
+    @Override
+    public LifecycleTransaction getFlushTransaction()
+    {
+        return flushTransaction.get();
+    }
+
+    @Override
+    public LifecycleTransaction setFlushTransaction(LifecycleTransaction flushTransaction)
+    {
+        return this.flushTransaction.getAndSet(flushTransaction);
+    }
+
     protected static class ColumnsCollector
     {
         private final HashMap<ColumnMetadata, AtomicBoolean> predefined = new HashMap<>();
@@ -205,26 +223,31 @@ public abstract class AbstractMemtable implements Memtable
 
     protected abstract class AbstractFlushablePartitionSet<P extends Partition> implements FlushablePartitionSet<P>
     {
+        @Override
         public long dataSize()
         {
             return getLiveDataSize();
         }
 
+        @Override
         public CommitLogPosition commitLogLowerBound()
         {
             return AbstractMemtable.this.getCommitLogLowerBound();
         }
 
+        @Override
         public LastCommitLogPosition commitLogUpperBound()
         {
             return AbstractMemtable.this.getFinalCommitLogUpperBound();
         }
 
+        @Override
         public EncodingStats encodingStats()
         {
             return AbstractMemtable.this.encodingStats();
         }
 
+        @Override
         public RegularAndStaticColumns columns()
         {
             return AbstractMemtable.this.columns();
diff --git a/src/java/org/apache/cassandra/db/memtable/Flushing.java b/src/java/org/apache/cassandra/db/memtable/Flushing.java
index afe7a12000..1d24277bbe 100644
--- a/src/java/org/apache/cassandra/db/memtable/Flushing.java
+++ b/src/java/org/apache/cassandra/db/memtable/Flushing.java
@@ -23,6 +23,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.Callable;
 
+import com.google.common.base.Preconditions;
 import com.google.common.base.Throwables;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -58,6 +59,12 @@ public class Flushing
                                                      Memtable memtable,
                                                      LifecycleTransaction txn)
     {
+        LifecycleTransaction ongoingFlushTransaction = memtable.setFlushTransaction(txn);
+        Preconditions.checkState(ongoingFlushTransaction == null,
+                                 "Attempted to flush Memtable more than once on %s.%s",
+                                 cfs.keyspace.getName(),
+                                 cfs.name);
+
         DiskBoundaries diskBoundaries = cfs.getDiskBoundaries();
         List<PartitionPosition> boundaries = diskBoundaries.positions;
         List<Directories.DataDirectory> locations = diskBoundaries.directories;
diff --git a/src/java/org/apache/cassandra/db/memtable/Memtable.java b/src/java/org/apache/cassandra/db/memtable/Memtable.java
index 3a18a8c7d7..603e3bc028 100644
--- a/src/java/org/apache/cassandra/db/memtable/Memtable.java
+++ b/src/java/org/apache/cassandra/db/memtable/Memtable.java
@@ -26,6 +26,7 @@ import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.db.RegularAndStaticColumns;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.partitions.Partition;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.EncodingStats;
@@ -379,6 +380,10 @@ public interface Memtable extends Comparable<Memtable>, UnfilteredSource
     /** True if the memtable contains no data */
     boolean isClean();
 
+    // The following two methods provide a way of tracking ongoing flushes
+    LifecycleTransaction setFlushTransaction(LifecycleTransaction transaction);
+    LifecycleTransaction getFlushTransaction();
+
     /** Order memtables by time as reflected in the commit log position at time of construction */
     default int compareTo(Memtable that)
     {
diff --git a/src/java/org/apache/cassandra/index/sai/IndexContext.java b/src/java/org/apache/cassandra/index/sai/IndexContext.java
index ca67f3d340..0a47728864 100644
--- a/src/java/org/apache/cassandra/index/sai/IndexContext.java
+++ b/src/java/org/apache/cassandra/index/sai/IndexContext.java
@@ -21,51 +21,52 @@ package org.apache.cassandra.index.sai;
 import java.nio.ByteBuffer;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Objects;
 import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
 import javax.annotation.Nullable;
 
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ImmutableSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.cql3.Operator;
 import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.db.ClusteringComparator;
 import org.apache.cassandra.db.DecoratedKey;
-import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.AsciiType;
 import org.apache.cassandra.db.marshal.BooleanType;
 import org.apache.cassandra.db.marshal.CompositeType;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.db.marshal.UUIDType;
-import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.rows.Cell;
 import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.dht.AbstractBounds;
 import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
-import org.apache.cassandra.index.sai.memory.MemtableIndex;
+import org.apache.cassandra.index.sai.disk.SSTableIndex;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.memory.MemtableIndexManager;
+import org.apache.cassandra.index.sai.metrics.ColumnQueryMetrics;
 import org.apache.cassandra.index.sai.metrics.IndexMetrics;
 import org.apache.cassandra.index.sai.plan.Expression;
 import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.index.sai.utils.PrimaryKeyFactory;
-import org.apache.cassandra.index.sai.utils.KeyRangeIterator;
-import org.apache.cassandra.index.sai.utils.KeyRangeUnionIterator;
 import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.index.sai.view.IndexViewManager;
+import org.apache.cassandra.index.sai.view.View;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.IndexMetadata;
-import org.apache.cassandra.utils.Clock;
-import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
 
 /**
- * Manage metadata for each column index.
+ * Manages metadata for each column index.
  */
 public class IndexContext
 {
+    private static final Logger logger = LoggerFactory.getLogger(IndexContext.class);
+
     private static final Set<AbstractType<?>> EQ_ONLY_TYPES = ImmutableSet.of(UTF8Type.instance,
                                                                               AsciiType.instance,
                                                                               BooleanType.instance,
@@ -82,15 +83,16 @@ public class IndexContext
 
     // Config can be null if the column context is "fake" (i.e. created for a filtering expression).
     @Nullable
-    private final IndexMetadata config;
+    private final IndexMetadata indexMetadata;
 
-    private final ConcurrentMap<Memtable, MemtableIndex> liveMemtableIndexMap;
+    private final MemtableIndexManager memtableIndexManager;
 
+    private final IndexViewManager viewManager;
     private final IndexMetrics indexMetrics;
-
+    private final ColumnQueryMetrics columnQueryMetrics;
     private final AbstractAnalyzer.AnalyzerFactory indexAnalyzerFactory;
     private final AbstractAnalyzer.AnalyzerFactory queryAnalyzerFactory;
-    private final PrimaryKeyFactory primaryKeyFactory;
+    private final PrimaryKey.Factory primaryKeyFactory;
 
     public IndexContext(String keyspace,
                         String table,
@@ -98,7 +100,7 @@ public class IndexContext
                         ClusteringComparator clusteringComparator,
                         ColumnMetadata columnMetadata,
                         IndexTarget.Type indexType,
-                        @Nullable IndexMetadata config)
+                        @Nullable IndexMetadata indexMetadata)
     {
         this.keyspace = Objects.requireNonNull(keyspace);
         this.table = Objects.requireNonNull(table);
@@ -107,11 +109,13 @@ public class IndexContext
         this.columnMetadata = Objects.requireNonNull(columnMetadata);
         this.indexType = Objects.requireNonNull(indexType);
         this.validator = TypeUtil.cellValueType(columnMetadata, indexType);
-        this.primaryKeyFactory = PrimaryKey.factory(clusteringComparator);
+        this.primaryKeyFactory = new PrimaryKey.Factory(clusteringComparator);
 
-        this.config = config;
-        this.indexMetrics = config == null ? null : new IndexMetrics(this);
-        this.liveMemtableIndexMap = config == null ? null : new ConcurrentHashMap<>();
+        this.indexMetadata = indexMetadata;
+        this.memtableIndexManager = indexMetadata == null ? null : new MemtableIndexManager(this);
+        this.indexMetrics = indexMetadata == null ? null : new IndexMetrics(this);
+        this.viewManager = new IndexViewManager(this);
+        this.columnQueryMetrics = new ColumnQueryMetrics.TrieIndexMetrics(this);
 
         // We currently only support the NoOpAnalyzer
         this.indexAnalyzerFactory = AbstractAnalyzer.fromOptions(getValidator(), Collections.emptyMap());
@@ -123,122 +127,42 @@ public class IndexContext
         return partitionKeyType;
     }
 
-    public PrimaryKeyFactory keyFactory()
+    public PrimaryKey.Factory keyFactory()
     {
         return primaryKeyFactory;
     }
 
-    public ClusteringComparator comparator()
-    {
-        return clusteringComparator;
-    }
-
     public String getKeyspace()
     {
         return keyspace;
     }
 
-    public String getTable()
+    public IndexMetrics getIndexMetrics()
     {
-        return table;
+        return indexMetrics;
     }
 
-    public IndexMetadata getIndexMetadata()
+    public ColumnQueryMetrics getColumnQueryMetrics()
     {
-        return config;
-    }
-
-    public long index(DecoratedKey key, Row row, Memtable mt)
-    {
-        assert config != null : "Attempt to index on a non-indexing context";
-
-        MemtableIndex current = liveMemtableIndexMap.get(mt);
-
-        // We expect the relevant IndexMemtable to be present most of the time, so only make the
-        // call to computeIfAbsent() if it's not. (see https://bugs.openjdk.java.net/browse/JDK-8161372)
-        MemtableIndex target = (current != null)
-                               ? current
-                               : liveMemtableIndexMap.computeIfAbsent(mt, memtable -> new MemtableIndex(this));
-
-        long start = Clock.Global.nanoTime();
-
-        long bytes = 0;
-
-        if (isNonFrozenCollection())
-        {
-            Iterator<ByteBuffer> bufferIterator = getValuesOf(row, FBUtilities.nowInSeconds());
-            if (bufferIterator != null)
-            {
-                while (bufferIterator.hasNext())
-                {
-                    ByteBuffer value = bufferIterator.next();
-                    bytes += target.index(key, row.clustering(), value);
-                }
-            }
-        }
-        else
-        {
-            ByteBuffer value = getValueOf(key, row, FBUtilities.nowInSeconds());
-            target.index(key, row.clustering(), value);
-        }
-        indexMetrics.memtableIndexWriteLatency.update(Clock.Global.nanoTime() - start, TimeUnit.NANOSECONDS);
-        return bytes;
+        return columnQueryMetrics;
     }
 
-    public void renewMemtable(Memtable renewed)
-    {
-        assert liveMemtableIndexMap != null : "Attempt to renew memtable on non-indexing context";
-
-        for (Memtable memtable : liveMemtableIndexMap.keySet())
-        {
-            // remove every index but the one that corresponds to the post-truncate Memtable
-            if (renewed != memtable)
-            {
-                liveMemtableIndexMap.remove(memtable);
-            }
-        }
-    }
-
-    public void discardMemtable(Memtable discarded)
-    {
-        assert liveMemtableIndexMap != null : "Attempt to discard memtable from non-indexing context";
-
-        liveMemtableIndexMap.remove(discarded);
-    }
-
-    public KeyRangeIterator searchMemtableIndexes(Expression e, AbstractBounds<PartitionPosition> keyRange)
+    public String getTable()
     {
-        assert liveMemtableIndexMap != null : "Attempt to perform search on non-indexing context";
-
-        Collection<MemtableIndex> memtableIndexes = liveMemtableIndexMap.values();
-
-        if (memtableIndexes.isEmpty())
-        {
-            return KeyRangeIterator.empty();
-        }
-
-        KeyRangeIterator.Builder builder = KeyRangeUnionIterator.builder(memtableIndexes.size());
-
-        for (MemtableIndex memtableIndex : memtableIndexes)
-        {
-            builder.add(memtableIndex.search(e, keyRange));
-        }
-
-        return builder.build();
+        return table;
     }
 
-    public long liveMemtableWriteCount()
+    public IndexMetadata getIndexMetadata()
     {
-        assert liveMemtableIndexMap != null : "Attempt to get metrics from non-indexing context";
-
-        return liveMemtableIndexMap.values().stream().mapToLong(MemtableIndex::writeCount).sum();
+        return indexMetadata;
     }
 
-    public long estimatedMemIndexMemoryUsed()
+    /**
+     * @return A set of SSTables which have attached to them invalid index components.
+     */
+    public Collection<SSTableContext> onSSTableChanged(Collection<SSTableReader> oldSSTables, Collection<SSTableContext> newSSTables, boolean validate)
     {
-        assert liveMemtableIndexMap != null : "Attempt to get metrics from non-indexing context";
-
-        return liveMemtableIndexMap.values().stream().mapToLong(MemtableIndex::estimatedMemoryUsed).sum();
+        return viewManager.update(oldSSTables, newSSTables, validate);
     }
 
     public ColumnMetadata getDefinition()
@@ -269,7 +193,7 @@ public class IndexContext
     @Nullable
     public String getIndexName()
     {
-        return config == null ? null : config.name;
+        return indexMetadata == null ? null : indexMetadata.name;
     }
 
     /**
@@ -292,26 +216,53 @@ public class IndexContext
         return queryAnalyzerFactory;
     }
 
-    public boolean isIndexed()
+    public View getView()
     {
-        return config != null;
+        return viewManager.getView();
+    }
+
+    public MemtableIndexManager getMemtableIndexManager()
+    {
+        assert memtableIndexManager != null : "Attempt to use memtable index manager on non-indexed context";
+
+        return memtableIndexManager;
+    }
+
+    /**
+     * @return total number of per-index open files
+     */
+    public int openPerIndexFiles()
+    {
+        return viewManager.getView().size() * Version.LATEST.onDiskFormat().openFilesPerColumnIndex(this);
+    }
+
+    public void drop(Collection<SSTableReader> sstablesToRebuild)
+    {
+        viewManager.drop(sstablesToRebuild);
+    }
+
+    public boolean isNotIndexed()
+    {
+        return indexMetadata == null;
     }
 
     /**
      * Called when index is dropped. Clear all live in-memory indexes and close
-     * analyzer factories.
+     * analyzer factories. Mark all {@link SSTableIndex} as released and per-column index files
+     * will be removed when in-flight queries are completed.
      */
     public void invalidate()
     {
-        if (liveMemtableIndexMap != null)
-            liveMemtableIndexMap.clear();
-        if (indexMetrics != null)
-            indexMetrics.release();
+        viewManager.invalidate();
         indexAnalyzerFactory.close();
         if (queryAnalyzerFactory != indexAnalyzerFactory)
-        {
             queryAnalyzerFactory.close();
-        }
+        if (memtableIndexManager != null)
+            memtableIndexManager.invalidate();
+        if (indexMetrics != null)
+            indexMetrics.release();
+        if (columnQueryMetrics != null)
+            columnQueryMetrics.release();
     }
 
     public boolean supports(Operator op)
@@ -422,7 +373,7 @@ public class IndexContext
 
         return Objects.equals(columnMetadata, other.columnMetadata) &&
                (indexType == other.indexType) &&
-               Objects.equals(config, other.config) &&
+               Objects.equals(indexMetadata, other.indexMetadata) &&
                Objects.equals(partitionKeyType, other.partitionKeyType) &&
                Objects.equals(clusteringComparator, other.clusteringComparator);
     }
@@ -430,15 +381,15 @@ public class IndexContext
     @Override
     public int hashCode()
     {
-        return Objects.hash(columnMetadata, indexType, config, partitionKeyType, clusteringComparator);
+        return Objects.hash(columnMetadata, indexType, indexMetadata, partitionKeyType, clusteringComparator);
     }
 
     /**
      * A helper method for constructing consistent log messages for specific column indexes.
-     *
+     * <p>
      * Example: For the index "idx" in keyspace "ks" on table "tb", calling this method with the raw message
      * "Flushing new index segment..." will produce...
-     *
+     * <p>
      * "[ks.tb.idx] Flushing new index segment..."
      *
      * @param message The raw content of a logging message, without information identifying it with an index.
@@ -448,6 +399,99 @@ public class IndexContext
     public String logMessage(String message)
     {
         // Index names are unique only within a keyspace.
-        return String.format("[%s.%s.%s] %s", keyspace, table, config == null ? "?" : config.name, message);
+        return String.format("[%s.%s.%s] %s", keyspace, table, indexMetadata == null ? "?" : indexMetadata.name, message);
+    }
+
+    /**
+     * @return the indexes that are built on the given SSTables on the left and corrupted indexes'
+     * corresponding contexts on the right
+     */
+    public Pair<Collection<SSTableIndex>, Collection<SSTableContext>> getBuiltIndexes(Collection<SSTableContext> sstableContexts, boolean validate)
+    {
+        Set<SSTableIndex> valid = new HashSet<>(sstableContexts.size());
+        Set<SSTableContext> invalid = new HashSet<>();
+
+        for (SSTableContext sstableContext : sstableContexts)
+        {
+            if (sstableContext.sstable.isMarkedCompacted())
+                continue;
+
+            if (!sstableContext.indexDescriptor.isPerColumnIndexBuildComplete(this))
+            {
+                logger.debug(logMessage("An on-disk index build for SSTable {} has not completed."), sstableContext.descriptor());
+                continue;
+            }
+
+            if (sstableContext.indexDescriptor.isIndexEmpty(this))
+            {
+                logger.debug(logMessage("No on-disk index was built for SSTable {} because the SSTable " +
+                                        "had no indexable rows for the index."), sstableContext.descriptor());
+                continue;
+            }
+
+            try
+            {
+                if (validate)
+                {
+                    if (!sstableContext.indexDescriptor.validatePerIndexComponents(this))
+                    {
+                        logger.warn(logMessage("Invalid per-column component for SSTable {}"), sstableContext.descriptor());
+                        invalid.add(sstableContext);
+                        continue;
+                    }
+                }
+
+                SSTableIndex index = sstableContext.newSSTableIndex(this);
+                logger.debug(logMessage("Successfully created index for SSTable {}."), sstableContext.descriptor());
+
+                // Try to add new index to the set, if set already has such index, we'll simply release and move on.
+                // This covers situation when SSTable collection has the same SSTable multiple
+                // times because we don't know what kind of collection it actually is.
+                if (!valid.add(index))
+                {
+                    index.release();
+                }
+            }
+            catch (Throwable e)
+            {
+                logger.warn(logMessage("Failed to update per-column components for SSTable {}"), sstableContext.descriptor(), e);
+                invalid.add(sstableContext);
+            }
+        }
+
+        return Pair.create(valid, invalid);
+    }
+
+    /**
+     * @return the number of indexed rows in this index (aka. a pair of term and rowId)
+     */
+    public long getCellCount()
+    {
+        return getView().getIndexes()
+                        .stream()
+                        .mapToLong(SSTableIndex::getRowCount)
+                        .sum();
+    }
+
+    /**
+     * @return the total size (in bytes) of per-column index components
+     */
+    public long diskUsage()
+    {
+        return getView().getIndexes()
+                        .stream()
+                        .mapToLong(SSTableIndex::sizeOfPerColumnComponents)
+                        .sum();
+    }
+
+    /**
+     * @return the total memory usage (in bytes) of per-column index on-disk data structure
+     */
+    public long indexFileCacheSize()
+    {
+        return getView().getIndexes()
+                        .stream()
+                        .mapToLong(SSTableIndex::indexFileCacheSize)
+                        .sum();
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/QueryContext.java b/src/java/org/apache/cassandra/index/sai/QueryContext.java
index 4fff4160d3..4996acbd97 100644
--- a/src/java/org/apache/cassandra/index/sai/QueryContext.java
+++ b/src/java/org/apache/cassandra/index/sai/QueryContext.java
@@ -40,10 +40,17 @@ public class QueryContext
 
     public final long executionQuotaNano;
 
+    public long sstablesHit = 0;
+    public long segmentsHit = 0;
     public long partitionsRead = 0;
     public long rowsFiltered = 0;
 
-    public long queryTimeouts = 0;
+    public long trieSegmentsHit = 0;
+
+    public long triePostingsSkips = 0;
+    public long triePostingsDecodes = 0;
+
+    public boolean queryTimedOut = false;
 
     public QueryContext(ReadCommand readCommand, long executionQuotaMs)
     {
@@ -61,7 +68,7 @@ public class QueryContext
     {
         if (totalQueryTimeNs() >= executionQuotaNano && !DISABLE_TIMEOUT)
         {
-            queryTimeouts++;
+            queryTimedOut = true;
             throw new QueryCancelledException(readCommand);
         }
     }
diff --git a/src/java/org/apache/cassandra/index/sai/SSTableContext.java b/src/java/org/apache/cassandra/index/sai/SSTableContext.java
new file mode 100644
index 0000000000..d0ff6372b6
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/SSTableContext.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai;
+
+import java.util.Collections;
+
+import com.google.common.base.Objects;
+
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.disk.SSTableIndex;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.concurrent.Ref;
+import org.apache.cassandra.utils.concurrent.RefCounted;
+import org.apache.cassandra.utils.concurrent.SharedCloseableImpl;
+
+/**
+ * An {@link SSTableContext} is created for an individual sstable and is shared across column indexes to track per-sstable
+ * index files.
+ * <p>
+ * The {@link SSTableContext} will be released when receiving a sstable removed notification, but its shared copies in
+ * individual {@link SSTableIndex}es will be released when in-flight read requests complete.
+ */
+public class SSTableContext extends SharedCloseableImpl
+{
+    public final SSTableReader sstable;
+    public final IndexDescriptor indexDescriptor;
+    public final PrimaryKeyMap.Factory primaryKeyMapFactory;
+
+    private SSTableContext(SSTableReader sstable,
+                           IndexDescriptor indexDescriptor,
+                           PrimaryKeyMap.Factory primaryKeyMapFactory,
+                           Cleanup cleanup)
+    {
+        super(cleanup);
+        this.sstable = sstable;
+        this.indexDescriptor = indexDescriptor;
+        this.primaryKeyMapFactory = primaryKeyMapFactory;
+    }
+
+    private SSTableContext(SSTableContext copy)
+    {
+        super(copy);
+        this.sstable = copy.sstable;
+        this.indexDescriptor = copy.indexDescriptor;
+        this.primaryKeyMapFactory = copy.primaryKeyMapFactory;
+    }
+
+    @SuppressWarnings({"resource", "RedundantSuppression"})
+    public static SSTableContext create(SSTableReader sstable)
+    {
+        Ref<? extends SSTableReader> sstableRef = null;
+        PrimaryKeyMap.Factory primaryKeyMapFactory = null;
+
+        IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable);
+        try
+        {
+            sstableRef = sstable.tryRef();
+
+            if (sstableRef == null)
+            {
+                throw new IllegalStateException("Couldn't acquire reference to the sstable: " + sstable);
+            }
+
+            primaryKeyMapFactory = indexDescriptor.newPrimaryKeyMapFactory(sstable);
+
+            Cleanup cleanup = new Cleanup(primaryKeyMapFactory, indexDescriptor, sstableRef);
+
+            return new SSTableContext(sstable, indexDescriptor, primaryKeyMapFactory, cleanup);
+        }
+        catch (Throwable t)
+        {
+            if (sstableRef != null)
+            {
+                sstableRef.release();
+            }
+
+            throw Throwables.unchecked(Throwables.close(t, Collections.singleton(primaryKeyMapFactory)));
+        }
+    }
+
+    @Override
+    public SSTableContext sharedCopy()
+    {
+        return new SSTableContext(this);
+    }
+
+    /**
+     * Returns a new {@link SSTableIndex} for a per-column index
+     */
+    public SSTableIndex newSSTableIndex(IndexContext indexContext)
+    {
+        return indexDescriptor.newSSTableIndex(this, indexContext);
+    }
+
+    /**
+     * @return descriptor of attached sstable
+     */
+    public Descriptor descriptor()
+    {
+        return sstable.descriptor;
+    }
+
+    /**
+     * @return disk usage (in bytes) of per-sstable index files
+     */
+    public long diskUsage()
+    {
+        return indexDescriptor.sizeOnDiskOfPerSSTableComponents();
+    }
+
+    /**
+     * @return number of open files per {@link SSTableContext} instance
+     */
+    public int openFilesPerSSTable()
+    {
+        return indexDescriptor.version.onDiskFormat().openFilesPerSSTableIndex();
+    }
+
+    @Override
+    public String toString()
+    {
+        return "SSTableContext{" +
+               "sstable=" + sstable.descriptor +
+               '}';
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        SSTableContext that = (SSTableContext) o;
+        return Objects.equal(sstable.descriptor, that.sstable.descriptor);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hashCode(sstable.descriptor.hashCode());
+    }
+
+    private static class Cleanup implements RefCounted.Tidy
+    {
+        private final PrimaryKeyMap.Factory primaryKeyMapFactory;
+        private final IndexDescriptor indexDescriptor;
+        private final Ref<? extends SSTableReader> sstableRef;
+
+        private Cleanup(PrimaryKeyMap.Factory primaryKeyMapFactory,
+                        IndexDescriptor indexDescriptor,
+                        Ref<? extends SSTableReader> sstableRef)
+        {
+            this.primaryKeyMapFactory = primaryKeyMapFactory;
+            this.indexDescriptor = indexDescriptor;
+            this.sstableRef = sstableRef;
+        }
+
+        @Override
+        public void tidy()
+        {
+            Throwable t = sstableRef.ensureReleased(null);
+            t = Throwables.close(t, Collections.singleton(primaryKeyMapFactory));
+
+            Throwables.maybeFail(t);
+        }
+
+        @Override
+        public String name()
+        {
+            return indexDescriptor.toString();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/SSTableContextManager.java b/src/java/org/apache/cassandra/index/sai/SSTableContextManager.java
new file mode 100644
index 0000000000..ba65806e86
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/SSTableContextManager.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import javax.annotation.concurrent.ThreadSafe;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.Pair;
+
+/**
+ * Manages per-sstable {@link SSTableContext}s for {@link StorageAttachedIndexGroup}
+ */
+@ThreadSafe
+public class SSTableContextManager
+{
+    private static final Logger logger = LoggerFactory.getLogger(SSTableContextManager.class);
+
+    private final ConcurrentHashMap<SSTableReader, SSTableContext> sstableContexts = new ConcurrentHashMap<>();
+
+    /**
+     * Initialize {@link SSTableContext}s if they are not already initialized.
+     *
+     * @param removed SSTables being removed
+     * @param added SSTables being added
+     * @param validate if true, header and footer will be validated.
+     *
+     * @return a set of contexts for SSTables with valid per-SSTable components, and a set of
+     * SSTables with invalid or missing components
+     */
+    public Pair<Set<SSTableContext>, Set<SSTableReader>> update(Collection<SSTableReader> removed, Iterable<SSTableReader> added, boolean validate)
+    {
+        release(removed);
+
+        Set<SSTableContext> contexts = new HashSet<>();
+        Set<SSTableReader> invalid = new HashSet<>();
+
+        for (SSTableReader sstable : added)
+        {
+            if (sstable.isMarkedCompacted())
+            {
+                continue;
+            }
+
+            IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable);
+
+            if (!indexDescriptor.isPerSSTableIndexBuildComplete())
+            {
+                // Don't even try to validate or add the context if the completion marker is missing.
+                continue;
+            }
+
+            try
+            {
+                // Only validate on restart or newly refreshed SSTable. Newly built files are unlikely to be corrupted.
+                if (validate && !sstableContexts.containsKey(sstable) && !indexDescriptor.validatePerSSTableComponents())
+                {
+                    logger.warn(indexDescriptor.logMessage("Invalid per-SSTable component for SSTable {}"), sstable.descriptor);
+                    invalid.add(sstable);
+                    removeInvalidSSTableContext(sstable);
+                    continue;
+                }
+                // ConcurrentHashMap#computeIfAbsent guarantees atomicity, so {@link SSTableContext#create(SSTableReader)}}
+                // is called at most once per key.
+                contexts.add(sstableContexts.computeIfAbsent(sstable, SSTableContext::create));
+            }
+            catch (Throwable t)
+            {
+                logger.warn(indexDescriptor.logMessage("Failed to update per-SSTable components for SSTable {}"), sstable.descriptor, t);
+                invalid.add(sstable);
+                removeInvalidSSTableContext(sstable);
+            }
+        }
+
+        return Pair.create(contexts, invalid);
+    }
+
+    public void release(Collection<SSTableReader> toRelease)
+    {
+        toRelease.stream().map(sstableContexts::remove).filter(Objects::nonNull).forEach(SSTableContext::close);
+    }
+
+    /**
+     * @return total number of per-sstable open files for live sstables
+     */
+    int openFiles()
+    {
+        return sstableContexts.values().stream().mapToInt(SSTableContext::openFilesPerSSTable).sum();
+    }
+
+    /**
+     * @return total disk usage (in bytes) of all per-sstable index files
+     */
+    long diskUsage()
+    {
+        return sstableContexts.values().stream().mapToLong(SSTableContext::diskUsage).sum();
+    }
+
+    Set<SSTableReader> sstables()
+    {
+        return sstableContexts.keySet();
+    }
+
+    @VisibleForTesting
+    public int size()
+    {
+        return sstableContexts.size();
+    }
+
+    @VisibleForTesting
+    public void clear()
+    {
+        sstableContexts.values().forEach(SSTableContext::close);
+        sstableContexts.clear();
+    }
+
+    @SuppressWarnings("EmptyTryBlock")
+    private void removeInvalidSSTableContext(SSTableReader sstable)
+    {
+        try (SSTableContext ignored = sstableContexts.remove(sstable))
+        {
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java
index 6c79e1f08b..8bb99fa04d 100644
--- a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java
+++ b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndex.java
@@ -18,21 +18,35 @@
 
 package org.apache.cassandra.index.sai;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
+import java.util.NavigableMap;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CompletableFuture; // checkstyle: permit this import
 import java.util.concurrent.Future;
+import java.util.function.BooleanSupplier;
+import java.util.stream.Collectors;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.util.concurrent.Futures; // checkstyle: permit this import
+import com.google.common.util.concurrent.ListenableFuture; // checkstyle: permit this import
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQL3Type;
 import org.apache.cassandra.cql3.Operator;
 import org.apache.cassandra.cql3.statements.schema.IndexTarget;
@@ -42,7 +56,7 @@ import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.ReadCommand;
 import org.apache.cassandra.db.RegularAndStaticColumns;
 import org.apache.cassandra.db.WriteContext;
-import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.filter.RowFilter;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.marshal.AbstractType;
@@ -60,14 +74,21 @@ import org.apache.cassandra.index.IndexRegistry;
 import org.apache.cassandra.index.SecondaryIndexBuilder;
 import org.apache.cassandra.index.TargetParser;
 import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sai.disk.SSTableIndex;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.format.Version;
 import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.index.sai.view.View;
 import org.apache.cassandra.index.transactions.IndexTransaction;
+import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.SSTableIdFactory;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.concurrent.OpOrder;
 
@@ -83,20 +104,34 @@ public class StorageAttachedIndex implements Index
                                                        Collection<SSTableReader> sstablesToRebuild,
                                                        boolean isFullRebuild)
         {
-            // Return an empty builder for in-memory implementation
-            return new SecondaryIndexBuilder()
-            {
-                @Override
-                public void build()
-                {
-                }
-
-                @Override
-                public CompactionInfo getCompactionInfo()
-                {
-                    return null;
-                }
-            };
+            NavigableMap<SSTableReader, Set<StorageAttachedIndex>> sstables = new TreeMap<>(Comparator.comparing(s -> s.descriptor.id, SSTableIdFactory.COMPARATOR));
+            StorageAttachedIndexGroup group = StorageAttachedIndexGroup.getIndexGroup(cfs);
+
+            assert group != null : "Index group does not exist for table " + cfs.keyspace + '.' + cfs.name;
+
+            indexes.stream()
+                   .filter((i) -> i instanceof StorageAttachedIndex)
+                   .forEach((i) ->
+                            {
+                                StorageAttachedIndex sai = (StorageAttachedIndex) i;
+                                IndexContext indexContext = ((StorageAttachedIndex) i).getIndexContext();
+
+                                // If this is not a full manual index rebuild we can skip SSTables that already have an
+                                // attached index. Otherwise, we override any pre-existent index.
+                                Collection<SSTableReader> ss = sstablesToRebuild;
+                                if (!isFullRebuild)
+                                {
+                                    ss = sstablesToRebuild.stream()
+                                                          .filter(s -> !IndexDescriptor.create(s).isPerColumnIndexBuildComplete(indexContext))
+                                                          .collect(Collectors.toList());
+                                }
+
+                                group.dropIndexSSTables(ss, sai);
+
+                                ss.forEach(sstable -> sstables.computeIfAbsent(sstable, ignore -> new HashSet<>()).add(sai));
+                            });
+
+            return new StorageAttachedIndexBuilder(group, sstables, isFullRebuild, false);
         }
     }
 
@@ -119,6 +154,12 @@ public class StorageAttachedIndex implements Index
     private final ColumnFamilyStore baseCfs;
     private final IndexContext indexContext;
 
+    // Tracks whether we've started the index build on initialization.
+    private volatile boolean initBuildStarted = false;
+
+    // Tracks whether the index has been invalidated due to removal, a table drop, etc.
+    private volatile boolean valid = true;
+
     public StorageAttachedIndex(ColumnFamilyStore baseCfs, IndexMetadata config)
     {
         this.baseCfs = baseCfs;
@@ -229,17 +270,96 @@ public class StorageAttachedIndex implements Index
     {
         // New storage-attached indexes will be available for queries after on disk index data are built.
         // Memtable data will be indexed via flushing triggered by schema change
-        return () -> startInitialBuild(baseCfs).get();
+        // We only want to validate the index files if we are starting up
+        return () -> startInitialBuild(baseCfs, StorageService.instance.isStarting()).get();
     }
 
-    private Future<?> startInitialBuild(ColumnFamilyStore baseCfs)
+    private Future<?> startInitialBuild(ColumnFamilyStore baseCfs, boolean validate)
     {
         if (baseCfs.indexManager.isIndexQueryable(this))
         {
-            logger.debug(indexContext.logMessage("Skipping validation and building in initialization task, as pre-join has already made the storage attached index queryable..."));
+            logger.debug(indexContext.logMessage("Skipping validation and building in initialization task, as pre-join has already made the storage-attached index queryable..."));
+            initBuildStarted = true;
+            return CompletableFuture.completedFuture(null);
+        }
+
+        // stop in-progress compaction tasks to prevent compacted sstable not being indexed.
+        logger.debug(indexContext.logMessage("Stopping active compactions to make sure all sstables are indexed after initial build."));
+        CompactionManager.instance.interruptCompactionFor(Collections.singleton(baseCfs.metadata()),
+                                                          ssTableReader -> true,
+                                                          true);
+
+        // Force another flush to make sure on disk index is generated for memtable data before marking it queryable.
+        // In the case of offline scrub, there are no live memtables.
+        if (!baseCfs.getTracker().getView().liveMemtables.isEmpty())
+        {
+            baseCfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.INDEX_BUILD_STARTED);
+        }
+
+        // It is now safe to flush indexes directly from flushing Memtables.
+        initBuildStarted = true;
+
+        StorageAttachedIndexGroup indexGroup = StorageAttachedIndexGroup.getIndexGroup(baseCfs);
+
+        assert indexGroup != null : "Index group does not exist for table " + baseCfs.keyspace + '.' + baseCfs.name;
+
+        List<SSTableReader> nonIndexed = findNonIndexedSSTables(baseCfs, indexGroup, validate);
+
+        if (nonIndexed.isEmpty())
+        {
+            return CompletableFuture.completedFuture(null);
+        }
+
+        // split sorted sstables into groups with similar size and build each group in separate compaction thread
+        List<List<SSTableReader>> groups = groupBySize(nonIndexed, DatabaseDescriptor.getConcurrentIndexBuilders());
+        List<ListenableFuture<?>> futures = new ArrayList<>();
+
+        for (List<SSTableReader> group : groups)
+        {
+            SortedMap<SSTableReader, Set<StorageAttachedIndex>> current = new TreeMap<>(Comparator.comparing(s -> s.descriptor.id, SSTableIdFactory.COMPARATOR));
+            group.forEach(sstable -> current.put(sstable, Collections.singleton(this)));
+
+            futures.add(CompactionManager.instance.submitIndexBuild(new StorageAttachedIndexBuilder(indexGroup, current, false, true)));
+        }
+
+        logger.info(indexContext.logMessage("Submitting {} parallel initial index builds over {} total sstables..."), futures.size(), nonIndexed.size());
+        return Futures.allAsList(futures);
+    }
+
+    /**
+     * Splits SSTables into groups of similar overall size.
+     *
+     * @param toRebuild a list of SSTables to split (Note that this list will be sorted in place!)
+     * @param parallelism an upper bound on the number of groups
+     *
+     * @return a {@link List} of SSTable groups, each represented as a {@link List} of {@link SSTableReader}
+     */
+    @VisibleForTesting
+    public static List<List<SSTableReader>> groupBySize(List<SSTableReader> toRebuild, int parallelism)
+    {
+        List<List<SSTableReader>> groups = new ArrayList<>();
+
+        toRebuild.sort(Comparator.comparingLong(SSTableReader::onDiskLength).reversed());
+        Iterator<SSTableReader> sortedSSTables = toRebuild.iterator();
+        double dataPerCompactor = toRebuild.stream().mapToLong(SSTableReader::onDiskLength).sum() * 1.0 / parallelism;
+
+        while (sortedSSTables.hasNext())
+        {
+            long sum = 0;
+            List<SSTableReader> current = new ArrayList<>();
+
+            while (sortedSSTables.hasNext() && sum < dataPerCompactor)
+            {
+                SSTableReader sstable = sortedSSTables.next();
+                sum += sstable.onDiskLength();
+                current.add(sstable);
+            }
+
+            assert !current.isEmpty();
+            groups.add(current);
         }
-        baseCfs.indexManager.makeIndexQueryable(this, Status.BUILD_SUCCEEDED);
-        return CompletableFuture.completedFuture(null);
+
+        return groups;
     }
 
     @Override
@@ -259,6 +379,14 @@ public class StorageAttachedIndex implements Index
     {
         return () ->
         {
+            // mark index as invalid, in-progress SSTableIndexWriters will abort
+            valid = false;
+
+            // in case of dropping table, SSTable indexes should already been removed by SSTableListChangedNotification.
+            Set<Component> toRemove = getComponents();
+            for (SSTableIndex sstableIndex : indexContext.getView().getIndexes())
+                sstableIndex.getSSTable().unregisterComponents(toRemove, baseCfs.getTracker());
+
             indexContext.invalidate();
             return null;
         };
@@ -277,14 +405,38 @@ public class StorageAttachedIndex implements Index
         return this::startPreJoinTask;
     }
 
+    public boolean isInitBuildStarted()
+    {
+        return initBuildStarted;
+    }
+
+    public BooleanSupplier isIndexValid()
+    {
+        return () -> valid;
+    }
+
     private Future<?> startPreJoinTask()
     {
         try
         {
             if (baseCfs.indexManager.isIndexQueryable(this))
+            {
                 logger.debug(indexContext.logMessage("Skipping validation in pre-join task, as the initialization task has already made the index queryable..."));
+                baseCfs.indexManager.makeIndexQueryable(this, Status.BUILD_SUCCEEDED);
+                return null;
+            }
+
+            StorageAttachedIndexGroup indexGroup = StorageAttachedIndexGroup.getIndexGroup(baseCfs);
+
+            assert indexGroup != null : "Index group does not exist for table";
 
-            baseCfs.indexManager.makeIndexQueryable(this, Status.BUILD_SUCCEEDED);
+            Collection<SSTableReader> nonIndexed = findNonIndexedSSTables(baseCfs, indexGroup, true);
+
+            if (nonIndexed.isEmpty())
+            {
+                // If the index is complete, mark it queryable before the node starts accepting requests:
+                baseCfs.indexManager.makeIndexQueryable(this, Status.BUILD_SUCCEEDED);
+            }
         }
         catch (Throwable t)
         {
@@ -358,6 +510,40 @@ public class StorageAttachedIndex implements Index
     public void validate(PartitionUpdate update) throws InvalidRequestException
     {}
 
+    /**
+     * This method is called by the startup tasks to find SSTables that don't have indexes. The method is
+     * synchronized so that the view is unchanged between validation and the selection of non-indexed SSTables.
+     *
+     * @return a list SSTables without attached indexes
+     */
+    private synchronized List<SSTableReader> findNonIndexedSSTables(ColumnFamilyStore baseCfs, StorageAttachedIndexGroup group, boolean validate)
+    {
+        Set<SSTableReader> sstables = baseCfs.getLiveSSTables();
+
+        // Initialize the SSTable indexes w/ valid existing components...
+        assert group != null : "Missing index group on " + baseCfs.name;
+        group.onSSTableChanged(Collections.emptyList(), sstables, Collections.singleton(this), validate);
+
+        // ...then identify and rebuild the SSTable indexes that are missing.
+        List<SSTableReader> nonIndexed = new ArrayList<>();
+        View view = indexContext.getView();
+
+        for (SSTableReader sstable : sstables)
+        {
+            // An SSTable is considered not indexed if:
+            //   1. The current view does not contain the SSTable
+            //   2. The SSTable is not marked compacted
+            //   3. The column index does not have a completion marker
+            if (!view.containsSSTable(sstable) && !sstable.isMarkedCompacted() &&
+                !IndexDescriptor.create(sstable).isPerColumnIndexBuildComplete(indexContext))
+            {
+                nonIndexed.add(sstable);
+            }
+        }
+
+        return nonIndexed;
+    }
+
     @Override
     public Searcher searcherFor(ReadCommand command) throws InvalidRequestException
     {
@@ -372,6 +558,16 @@ public class StorageAttachedIndex implements Index
         throw new UnsupportedOperationException("Storage-attached index flush observers should never be created directly.");
     }
 
+    @Override
+    public Set<Component> getComponents()
+    {
+        return Version.LATEST.onDiskFormat()
+                             .perColumnIndexComponents(indexContext)
+                             .stream()
+                             .map(c -> Version.LATEST.makePerIndexComponent(c, indexContext))
+                             .collect(Collectors.toSet());
+    }
+
     @Override
     public Indexer indexerFor(DecoratedKey key,
                               RegularAndStaticColumns columns,
@@ -432,7 +628,8 @@ public class StorageAttachedIndex implements Index
         @Override
         public void insertRow(Row row)
         {
-            adjustMemtableSize(indexContext.index(key, row, memtable), CassandraWriteContext.fromContext(writeContext).getGroup());
+            adjustMemtableSize(indexContext.getMemtableIndexManager().index(key, row, memtable),
+                               CassandraWriteContext.fromContext(writeContext).getGroup());
         }
 
         @Override
diff --git a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuilder.java b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuilder.java
new file mode 100644
index 0000000000..8908b83df0
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuilder.java
@@ -0,0 +1,352 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.cassandra.index.sai;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.compaction.CompactionInfo;
+import org.apache.cassandra.db.compaction.CompactionInterruptedException;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.lifecycle.Tracker;
+import org.apache.cassandra.index.SecondaryIndexBuilder;
+import org.apache.cassandra.index.sai.disk.StorageAttachedIndexWriter;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.KeyIterator;
+import org.apache.cassandra.io.sstable.SSTableIdentityIterator;
+import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.TimeUUID;
+import org.apache.cassandra.utils.concurrent.CountDownLatch;
+import org.apache.cassandra.utils.concurrent.Ref;
+
+import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
+
+/**
+ * Multiple storage-attached indexes can start building concurrently. We need to make sure:
+ * 1. Per-SSTable index files are built only once
+ *      a. Per-SSTable index files already built, do nothing
+ *      b. Per-SSTable index files are currently building, we need to wait until it's built in order to consider index built.
+ * 2. Per-column index files are built for each column index
+ */
+public class StorageAttachedIndexBuilder extends SecondaryIndexBuilder
+{
+    protected static final Logger logger = LoggerFactory.getLogger(StorageAttachedIndexBuilder.class);
+
+    // make sure only one builder can write to per sstable files when multiple storage-attached indexes are created simultaneously.
+    private static final Map<SSTableReader, CountDownLatch> inProgress = Maps.newConcurrentMap();
+
+    private final StorageAttachedIndexGroup group;
+    private final TableMetadata metadata;
+    private final Tracker tracker;
+    private final TimeUUID compactionId = nextTimeUUID();
+    private final boolean isFullRebuild;
+    private final boolean isInitialBuild;
+
+    private final SortedMap<SSTableReader, Set<StorageAttachedIndex>> sstables;
+
+    private long bytesProcessed = 0;
+    private final long totalSizeInBytes;
+
+    StorageAttachedIndexBuilder(StorageAttachedIndexGroup group,
+                                SortedMap<SSTableReader, Set<StorageAttachedIndex>> sstables,
+                                boolean isFullRebuild,
+                                boolean isInitialBuild)
+    {
+        this.group = group;
+        this.metadata = group.metadata();
+        this.sstables = sstables;
+        this.tracker = group.table().getTracker();
+        this.isFullRebuild = isFullRebuild;
+        this.isInitialBuild = isInitialBuild;
+        this.totalSizeInBytes = sstables.keySet().stream().mapToLong(SSTableReader::uncompressedLength).sum();
+    }
+
+    @Override
+    public void build()
+    {
+        logger.debug(logMessage("Starting full index build"));
+        for (Map.Entry<SSTableReader, Set<StorageAttachedIndex>> e : sstables.entrySet())
+        {
+            SSTableReader sstable = e.getKey();
+            Set<StorageAttachedIndex> indexes = e.getValue();
+
+            Set<StorageAttachedIndex> existing = validateIndexes(indexes, sstable.descriptor);
+            if (existing.isEmpty())
+            {
+                logger.debug(logMessage("{} dropped during index build"), indexes);
+                continue;
+            }
+
+            if (indexSSTable(sstable, existing))
+            {
+                return;
+            }
+        }
+    }
+
+    private String logMessage(String message)
+    {
+        return String.format("[%s.%s.*] %s", metadata.keyspace, metadata.name, message);
+    }
+
+    /**
+     * @return true if index build should be stopped
+     */
+    private boolean indexSSTable(SSTableReader sstable, Set<StorageAttachedIndex> indexes)
+    {
+        logger.debug(logMessage("Starting index build on {}"), sstable.descriptor);
+
+        CountDownLatch perSSTableFileLock = null;
+        StorageAttachedIndexWriter indexWriter = null;
+
+        Ref<? extends SSTableReader> ref = sstable.tryRef();
+        if (ref == null)
+        {
+            logger.warn(logMessage("Couldn't acquire reference to the SSTable {}. It may have been removed."), sstable.descriptor);
+            return false;
+        }
+
+        try (RandomAccessReader dataFile = sstable.openDataReader();
+             LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.INDEX_BUILD, sstable))
+        {
+            perSSTableFileLock = shouldWritePerSSTableFiles(sstable);
+            // If we were unable to get the per-SSTable file lock it means that the
+            // per-SSTable components are already being built, so we only want to
+            // build the per-index components
+            boolean perIndexComponentsOnly = perSSTableFileLock == null;
+            // remove existing per column index files instead of overwriting
+            IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable);
+            indexes.forEach(index -> indexDescriptor.deleteColumnIndex(index.getIndexContext()));
+
+            indexWriter = StorageAttachedIndexWriter.createBuilderWriter(indexDescriptor, indexes, txn, perIndexComponentsOnly);
+
+            indexWriter.begin();
+
+            long previousBytesRead = 0;
+
+            try (KeyIterator keys = sstable.keyIterator())
+            {
+                while (keys.hasNext())
+                {
+                    if (isStopRequested())
+                    {
+                        logger.debug(indexDescriptor.logMessage("Index build has been stopped"));
+                        throw new CompactionInterruptedException(getCompactionInfo());
+                    }
+
+                    DecoratedKey key = keys.next();
+
+                    indexWriter.startPartition(key, -1, -1);
+
+                    long position = sstable.getPosition(key, SSTableReader.Operator.EQ);
+                    dataFile.seek(position);
+                    ByteBufferUtil.readWithShortLength(dataFile); // key
+
+                    try (SSTableIdentityIterator partition = SSTableIdentityIterator.create(sstable, dataFile, key))
+                    {
+                        // if the row has statics attached, it has to be indexed separately
+                        if (metadata.hasStaticColumns())
+                            indexWriter.nextUnfilteredCluster(partition.staticRow());
+
+                        while (partition.hasNext())
+                            indexWriter.nextUnfilteredCluster(partition.next());
+                    }
+                    long bytesRead = keys.getBytesRead();
+                    bytesProcessed += bytesRead - previousBytesRead;
+                    previousBytesRead = bytesRead;
+                }
+
+                completeSSTable(indexWriter, sstable, indexes, perSSTableFileLock);
+            }
+
+            return false;
+        }
+        catch (Throwable t)
+        {
+            if (indexWriter != null)
+            {
+                indexWriter.abort(t, true);
+            }
+
+            if (t instanceof InterruptedException)
+            {
+                // TODO: Is there anything that makes more sense than just restoring the interrupt?
+                logger.warn(logMessage("Interrupted while building indexes {} on SSTable {}"), indexes, sstable.descriptor);
+                Thread.currentThread().interrupt();
+                return true;
+            }
+            else if (t instanceof CompactionInterruptedException)
+            {
+                //TODO Shouldn't do this if the stop was interrupted by a truncate
+                if (isInitialBuild)
+                {
+                    logger.error(logMessage("Stop requested while building initial indexes {} on SSTable {}."), indexes, sstable.descriptor);
+                    throw Throwables.unchecked(t);
+                }
+                else
+                {
+                    logger.info(logMessage("Stop requested while building indexes {} on SSTable {}."), indexes, sstable.descriptor);
+                    return true;
+                }
+            }
+            else
+            {
+                logger.error(logMessage("Unable to build indexes {} on SSTable {}. Cause: {}."), indexes, sstable, t.getMessage());
+                throw Throwables.unchecked(t);
+            }
+        }
+        finally
+        {
+            ref.release();
+            // release current lock in case of error
+            if (perSSTableFileLock != null)
+            {
+                inProgress.remove(sstable);
+                perSSTableFileLock.decrement();
+            }
+        }
+    }
+
+    @Override
+    public CompactionInfo getCompactionInfo()
+    {
+        return new CompactionInfo(metadata,
+                                  OperationType.INDEX_BUILD,
+                                  bytesProcessed,
+                                  totalSizeInBytes,
+                                  compactionId,
+                                  sstables.keySet());
+    }
+
+    /**
+     * if the per sstable index files are already created, no need to write them again, unless found corrupted on rebuild
+     * if not created, try to acquire a lock, so only one builder will generate per sstable index files
+     */
+    private CountDownLatch shouldWritePerSSTableFiles(SSTableReader sstable)
+    {
+        // if per-table files are incomplete or checksum failed during full rebuild.
+        IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable);
+        if (!indexDescriptor.isPerSSTableIndexBuildComplete() ||
+            (isFullRebuild && !indexDescriptor.validatePerSSTableComponentsChecksum()))
+        {
+            CountDownLatch latch = CountDownLatch.newCountDownLatch(1);
+            if (inProgress.putIfAbsent(sstable, latch) == null)
+            {
+                // lock owner should clean up existing per-SSTable files
+                group.deletePerSSTableFiles(Collections.singleton(sstable));
+                return latch;
+            }
+        }
+        return null;
+    }
+
+    private void completeSSTable(SSTableFlushObserver indexWriter,
+                                 SSTableReader sstable,
+                                 Set<StorageAttachedIndex> indexes,
+                                 CountDownLatch latch) throws InterruptedException
+    {
+        indexWriter.complete();
+
+        if (latch != null)
+        {
+            // current builder owns the lock
+            latch.decrement();
+        }
+        else
+        {
+            /*
+             * When there is no lock, it means the per sstable index files are already created, just proceed to finish.
+             * When there is a lock held by another builder, wait for it to finish before finishing marking current index built.
+             */
+            latch = inProgress.get(sstable);
+            if (latch != null)
+                latch.await();
+        }
+
+        Set<StorageAttachedIndex> existing = validateIndexes(indexes, sstable.descriptor);
+        if (existing.isEmpty())
+        {
+            logger.debug(logMessage("{} dropped during index build"), indexes);
+            return;
+        }
+
+        // register custom index components into existing sstables
+        sstable.registerComponents(StorageAttachedIndexGroup.getLiveComponents(sstable, existing), tracker);
+        Set<StorageAttachedIndex> incomplete = group.onSSTableChanged(Collections.emptyList(), Collections.singleton(sstable), existing, false);
+
+        if (!incomplete.isEmpty())
+        {
+            // If this occurs during an initial index build, there is only one index in play, and
+            // throwing here to terminate makes sense. (This allows the initialization task to fail
+            // correctly and be marked as failed by the SIM.) In other cases, such as rebuilding a
+            // set of indexes for a new added/streamed SSTables, we terminate pessimistically. In
+            // other words, we abort the SSTable index write across all column indexes and mark
+            // then non-queryable until a restart or other incremental rebuild occurs.
+            throw new RuntimeException(logMessage("Failed to update views on column indexes " + incomplete + " on indexes " + indexes + '.'));
+        }
+    }
+
+    /**
+     *  In case of full rebuild, stop the index build if any index is dropped.
+     *  Otherwise, skip dropped indexes to avoid exception during repair/streaming.
+     */
+    private Set<StorageAttachedIndex> validateIndexes(Set<StorageAttachedIndex> indexes, Descriptor descriptor)
+    {
+        Set<StorageAttachedIndex> existing = new HashSet<>();
+        Set<StorageAttachedIndex> dropped = new HashSet<>();
+
+        for (StorageAttachedIndex index : indexes)
+        {
+            if (group.containsIndex(index))
+                existing.add(index);
+            else
+                dropped.add(index);
+        }
+
+        if (!dropped.isEmpty())
+        {
+            String droppedIndexes = dropped.stream().map(sai -> sai.getIndexContext().getIndexName()).collect(Collectors.toList()).toString();
+            if (isFullRebuild)
+                throw new RuntimeException(logMessage(String.format("%s are dropped, will stop index build.", droppedIndexes)));
+            else
+                logger.debug(logMessage("Skip building dropped index {} on sstable {}"), droppedIndexes, descriptor.baseFile());
+        }
+
+        return existing;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexGroup.java b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexGroup.java
index 8871f5dc19..0c98f2320a 100644
--- a/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexGroup.java
+++ b/src/java/org/apache/cassandra/index/sai/StorageAttachedIndexGroup.java
@@ -19,6 +19,7 @@ package org.apache.cassandra.index.sai;
 
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.Objects;
 import java.util.Set;
 import java.util.function.Predicate;
@@ -27,8 +28,11 @@ import javax.annotation.Nullable;
 import javax.annotation.concurrent.ThreadSafe;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
 import com.google.common.primitives.Ints;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
@@ -40,6 +44,11 @@ import org.apache.cassandra.db.lifecycle.Tracker;
 import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.sai.disk.SSTableIndex;
+import org.apache.cassandra.index.sai.disk.StorageAttachedIndexWriter;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.metrics.IndexGroupMetrics;
 import org.apache.cassandra.index.sai.metrics.TableQueryMetrics;
 import org.apache.cassandra.index.sai.metrics.TableStateMetrics;
 import org.apache.cassandra.index.sai.plan.StorageAttachedIndexQueryPlan;
@@ -47,11 +56,16 @@ import org.apache.cassandra.index.transactions.IndexTransaction;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.notifications.INotification;
 import org.apache.cassandra.notifications.INotificationConsumer;
 import org.apache.cassandra.notifications.MemtableDiscardedNotification;
 import org.apache.cassandra.notifications.MemtableRenewedNotification;
+import org.apache.cassandra.notifications.SSTableAddedNotification;
+import org.apache.cassandra.notifications.SSTableListChangedNotification;
 import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.Throwables;
 
 /**
  * Orchestrates building of storage-attached indices, and manages lifecycle of resources shared between them.
@@ -59,16 +73,23 @@ import org.apache.cassandra.schema.TableMetadata;
 @ThreadSafe
 public class StorageAttachedIndexGroup implements Index.Group, INotificationConsumer
 {
+    private static final Logger logger = LoggerFactory.getLogger(StorageAttachedIndexGroup.class);
+
     private final TableQueryMetrics queryMetrics;
     private final TableStateMetrics stateMetrics;
-    private final Set<Index> indexes = Sets.newConcurrentHashSet();
+    private final IndexGroupMetrics groupMetrics;
+    private final Set<StorageAttachedIndex> indexes = Sets.newConcurrentHashSet();
     private final ColumnFamilyStore baseCfs;
 
+    private final SSTableContextManager contextManager;
+
     StorageAttachedIndexGroup(ColumnFamilyStore baseCfs)
     {
         this.baseCfs = baseCfs;
         this.queryMetrics = new TableQueryMetrics(baseCfs.metadata());
         this.stateMetrics = new TableStateMetrics(baseCfs.metadata(), this);
+        this.groupMetrics = new IndexGroupMetrics(baseCfs.metadata(), this);
+        this.contextManager = new SSTableContextManager();
 
         Tracker tracker = baseCfs.getTracker();
         tracker.subscribe(this);
@@ -83,14 +104,14 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons
     @Override
     public Set<Index> getIndexes()
     {
-        return indexes;
+        return ImmutableSet.copyOf(indexes);
     }
 
     @Override
     public void addIndex(Index index)
     {
         assert index instanceof StorageAttachedIndex;
-        indexes.add(index);
+        indexes.add((StorageAttachedIndex) index);
     }
 
     @Override
@@ -100,10 +121,13 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons
         boolean removed = indexes.remove(index);
         assert removed : "Cannot remove non-existing index " + index;
         /*
-         * For the in-memory implementation we only need to unsubscribe from the tracker
+         * per index files are dropped via {@link StorageAttachedIndex#getInvalidateTask()}
          */
         if (indexes.isEmpty())
         {
+            for (SSTableReader sstable : contextManager.sstables())
+                sstable.unregisterComponents(IndexDescriptor.create(sstable).getLivePerSSTableComponents(), baseCfs.getTracker());
+            deletePerSSTableFiles(baseCfs.getLiveSSTables());
             baseCfs.getTracker().unsubscribe(this);
         }
     }
@@ -113,11 +137,13 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons
     {
         // in case of dropping table, sstable contexts should already been removed by SSTableListChangedNotification.
         queryMetrics.release();
+        groupMetrics.release();
         stateMetrics.release();
         baseCfs.getTracker().unsubscribe(this);
     }
 
     @Override
+    @SuppressWarnings("SuspiciousMethodCalls")
     public boolean containsIndex(Index index)
     {
         return indexes.contains(index);
@@ -165,7 +191,19 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons
     @Override
     public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata)
     {
-        return null;
+        IndexDescriptor indexDescriptor = IndexDescriptor.create(descriptor, tableMetadata.comparator);
+        try
+        {
+            return StorageAttachedIndexWriter.createFlushObserverWriter(indexDescriptor, indexes, tracker);
+        }
+        catch (Throwable t)
+        {
+            String message = "Unable to create storage-attached index writer on SSTable flush." +
+                             " All indexes from this table are going to be marked as non-queryable and will need to be rebuilt.";
+            logger.error(indexDescriptor.logMessage(message), t);
+            indexes.forEach(StorageAttachedIndex::makeIndexNonQueryable);
+            return null;
+        }
     }
 
     @Override
@@ -181,23 +219,147 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons
         return getComponents(indexes);
     }
 
-    // TODO: Currently returns an empty set for in-memory only implementation
-    static Set<Component> getComponents(Collection<Index> indices)
+    static Set<Component> getComponents(Collection<StorageAttachedIndex> indices)
+    {
+        Set<Component> components = Version.LATEST.onDiskFormat()
+                                                  .perSSTableIndexComponents()
+                                                  .stream()
+                                                  .map(Version.LATEST::makePerSSTableComponent)
+                                                  .collect(Collectors.toSet());
+        indices.forEach(index -> components.addAll(index.getComponents()));
+        return components;
+    }
+
+    // This differs from getComponents in that it only returns index components that exist on disk.
+    // It avoids errors being logged by the SSTable.readTOC method when we have an empty index.
+    @VisibleForTesting
+    public static Set<Component> getLiveComponents(SSTableReader sstable, Collection<StorageAttachedIndex> indices)
     {
-        return Collections.emptySet();
+        IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable);
+        Set<Component> components = indexDescriptor.getLivePerSSTableComponents();
+        indices.forEach(index -> components.addAll(indexDescriptor.getLivePerIndexComponents(index.getIndexContext())));
+        return components;
     }
 
     @Override
     public void handleNotification(INotification notification, Object sender)
     {
-        if (notification instanceof MemtableRenewedNotification)
+        // unfortunately, we can only check the type of notification via instanceof :(
+        if (notification instanceof SSTableAddedNotification)
         {
-            indexes.forEach(index -> ((StorageAttachedIndex)index).getIndexContext().renewMemtable(((MemtableRenewedNotification) notification).renewed));
+            SSTableAddedNotification notice = (SSTableAddedNotification) notification;
+
+            // Avoid validation for index files just written following Memtable flush.
+            boolean validate = !notice.memtable().isPresent();
+            onSSTableChanged(Collections.emptySet(), notice.added, indexes, validate);
+        }
+        else if (notification instanceof SSTableListChangedNotification)
+        {
+            SSTableListChangedNotification notice = (SSTableListChangedNotification) notification;
+
+            // Avoid validation for index files just written during compaction.
+            onSSTableChanged(notice.removed, notice.added, indexes, false);
+        }
+        else if (notification instanceof MemtableRenewedNotification)
+        {
+            indexes.forEach(index -> index.getIndexContext().getMemtableIndexManager().renewMemtable(((MemtableRenewedNotification) notification).renewed));
         }
         else if (notification instanceof MemtableDiscardedNotification)
         {
-            indexes.forEach(index -> ((StorageAttachedIndex)index).getIndexContext().discardMemtable(((MemtableDiscardedNotification) notification).memtable));
+            indexes.forEach(index -> index.getIndexContext().getMemtableIndexManager().discardMemtable(((MemtableDiscardedNotification) notification).memtable));
+        }
+    }
+
+    void deletePerSSTableFiles(Collection<SSTableReader> sstables)
+    {
+        contextManager.release(sstables);
+        sstables.forEach(sstableReader -> IndexDescriptor.create(sstableReader).deletePerSSTableIndexComponents());
+    }
+
+    void dropIndexSSTables(Collection<SSTableReader> ss, StorageAttachedIndex index)
+    {
+        try
+        {
+            index.getIndexContext().drop(ss);
+        }
+        catch (Throwable t)
+        {
+            // Mark the index non-queryable, as its view may be compromised.
+            index.makeIndexNonQueryable();
+
+            throw Throwables.unchecked(t);
+        }
+    }
+
+    /**
+     * This method is synchronized to avoid concurrent initialization tasks validating same per-SSTable files.
+     *
+     * @return the set of column indexes that were marked as non-queryable as a result of their per-SSTable index
+     * files being corrupt or being unable to successfully update their views
+     */
+    synchronized Set<StorageAttachedIndex> onSSTableChanged(Collection<SSTableReader> removed, Iterable<SSTableReader> added,
+                                                            Set<StorageAttachedIndex> indexes, boolean validate)
+    {
+        Pair<Set<SSTableContext>, Set<SSTableReader>> results = contextManager.update(removed, added, validate);
+
+        if (!results.right.isEmpty())
+        {
+            results.right.forEach(sstable -> {
+                IndexDescriptor indexDescriptor = IndexDescriptor.create(sstable);
+                indexDescriptor.deletePerSSTableIndexComponents();
+                // Column indexes are invalid if their SSTable-level components are corrupted so delete
+                // their associated index files and mark them non-queryable.
+                indexes.forEach(index -> {
+                    indexDescriptor.deleteColumnIndex(index.getIndexContext());
+                    index.makeIndexNonQueryable();
+                });
+            });
+            return indexes;
+        }
+
+        Set<StorageAttachedIndex> incomplete = new HashSet<>();
+
+        for (StorageAttachedIndex index : indexes)
+        {
+            Collection<SSTableContext> invalid = index.getIndexContext().onSSTableChanged(removed, results.left, validate);
+
+            if (!invalid.isEmpty())
+            {
+                // Delete the index files and mark the index non-queryable, as its view may be compromised,
+                // and incomplete, for our callers:
+                invalid.forEach(context -> context.indexDescriptor.deleteColumnIndex(index.getIndexContext()));
+                index.makeIndexNonQueryable();
+                incomplete.add(index);
+            }
         }
+        return incomplete;
+    }
+
+    /**
+     * open index files by checking number of {@link SSTableContext} and {@link SSTableIndex},
+     * so transient open files during validation and files that are still open for in-flight requests will not be tracked.
+     *
+     * @return total number of open files for all {@link StorageAttachedIndex}es.
+     */
+    public int openIndexFiles()
+    {
+        return contextManager.openFiles() + indexes.stream().mapToInt(index -> index.getIndexContext().openPerIndexFiles()).sum();
+    }
+
+    /**
+     * @return total disk usage (in bytes) of all per-sstable index files
+     */
+    public long diskUsage()
+    {
+        return contextManager.diskUsage();
+    }
+
+    /**
+     * @return count of indexes building
+     */
+    public int totalIndexBuildsInProgress()
+    {
+        return (int) indexes.stream().filter(i -> baseCfs.indexManager.isIndexBuilding(i.getIndexMetadata().name)).count();
     }
 
     /**
@@ -216,6 +378,15 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons
         return indexes.size();
     }
 
+    /**
+     * @return total disk usage of all per-sstable index files and per-column index files
+     */
+    public long totalDiskUsage()
+    {
+        return diskUsage() + indexes.stream().flatMap(index -> index.getIndexContext().getView().getIndexes().stream())
+                                    .mapToLong(SSTableIndex::sizeOfPerColumnComponents).sum();
+    }
+
     public TableMetadata metadata()
     {
         return baseCfs.metadata();
@@ -226,12 +397,31 @@ public class StorageAttachedIndexGroup implements Index.Group, INotificationCons
         return baseCfs;
     }
 
+    @VisibleForTesting
+    public SSTableContextManager sstableContextManager()
+    {
+        return contextManager;
+    }
+
+    /**
+     * simulate index loading on restart with index file validation
+     */
+    @VisibleForTesting
+    public void unsafeReload()
+    {
+        contextManager.clear();
+        onSSTableChanged(baseCfs.getLiveSSTables(), Collections.emptySet(), indexes, false);
+        onSSTableChanged(Collections.emptySet(), baseCfs.getLiveSSTables(), indexes, true);
+    }
+
     /**
      * Simulate the index going through a restart of node
      */
     @VisibleForTesting
     public void reset()
     {
-        indexes.forEach(i -> ((StorageAttachedIndex) i).makeIndexNonQueryable());
+        contextManager.clear();
+        indexes.forEach(StorageAttachedIndex::makeIndexNonQueryable);
+        onSSTableChanged(baseCfs.getLiveSSTables(), Collections.emptySet(), indexes, false);
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java b/src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java
index 64a2f2fed5..d1e7bae2fe 100644
--- a/src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java
+++ b/src/java/org/apache/cassandra/index/sai/analyzer/AbstractAnalyzer.java
@@ -28,7 +28,6 @@ import org.apache.cassandra.db.marshal.AbstractType;
 public abstract class AbstractAnalyzer implements Iterator<ByteBuffer>
 {
     protected ByteBuffer next = null;
-    String nextLiteral = null;
 
     /**
      * @return true if index value is transformed, e.g. normalized or lower-cased or tokenized.
@@ -66,7 +65,6 @@ public abstract class AbstractAnalyzer implements Iterator<ByteBuffer>
     public void reset(ByteBuffer input)
     {
         this.next = null;
-        this.nextLiteral = null;
 
         resetInternal(input);
     }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/IndexSearchResultIterator.java b/src/java/org/apache/cassandra/index/sai/disk/IndexSearchResultIterator.java
new file mode 100644
index 0000000000..f10bf5b8ae
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/IndexSearchResultIterator.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.exceptions.QueryCancelledException;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.iterators.KeyRangeUnionIterator;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.Throwables;
+
+public class IndexSearchResultIterator extends KeyRangeIterator
+{
+    private static final Logger logger = LoggerFactory.getLogger(IndexSearchResultIterator.class);
+
+    private final QueryContext context;
+    private final KeyRangeIterator union;
+    private final Collection<SSTableIndex> referencedIndexes;
+
+    private IndexSearchResultIterator(KeyRangeIterator union, Collection<SSTableIndex> referencedIndexes, QueryContext queryContext)
+    {
+        super(union.getMinimum(), union.getMaximum(), union.getCount());
+
+        this.union = union;
+        this.referencedIndexes = referencedIndexes;
+        this.context = queryContext;
+    }
+
+    /**
+     * Builds a new {@link IndexSearchResultIterator} that wraps a {@link KeyRangeUnionIterator} over the
+     * results of searching the {@link org.apache.cassandra.index.sai.memory.MemtableIndex} and the {@link SSTableIndex}es.
+     */
+    @SuppressWarnings({"resource", "RedundantSuppression"})
+    public static IndexSearchResultIterator build(Expression expression,
+                                                  Collection<SSTableIndex> sstableIndexes,
+                                                  AbstractBounds<PartitionPosition> keyRange,
+                                                  QueryContext queryContext)
+    {
+        List<KeyRangeIterator> subIterators = new ArrayList<>(1 + sstableIndexes.size());
+
+        KeyRangeIterator memtableIterator = expression.context.getMemtableIndexManager().searchMemtableIndexes(expression, keyRange);
+        if (memtableIterator != null)
+            subIterators.add(memtableIterator);
+
+        for (SSTableIndex sstableIndex : sstableIndexes)
+        {
+            try
+            {
+                queryContext.checkpoint();
+                queryContext.sstablesHit++;
+
+                if (sstableIndex.isReleased())
+                    throw new IllegalStateException(sstableIndex.getIndexContext().logMessage("Index was released from the view during the query"));
+
+                List<KeyRangeIterator> segmentIterators = sstableIndex.search(expression, keyRange, queryContext);
+
+                if (!segmentIterators.isEmpty())
+                    subIterators.addAll(segmentIterators);
+            }
+            catch (Throwable e)
+            {
+                if (!(e instanceof QueryCancelledException))
+                    logger.debug(sstableIndex.getIndexContext().logMessage(String.format("Failed search an index %s, aborting query.", sstableIndex.getSSTable())), e);
+
+                throw Throwables.cleaned(e);
+            }
+        }
+
+        KeyRangeIterator union = KeyRangeUnionIterator.build(subIterators);
+        return new IndexSearchResultIterator(union, sstableIndexes, queryContext);
+    }
+
+    protected PrimaryKey computeNext()
+    {
+        try
+        {
+            return union.hasNext() ? union.next() : endOfData();
+        }
+        finally
+        {
+            context.checkpoint();
+        }
+    }
+
+    protected void performSkipTo(PrimaryKey nextKey)
+    {
+        try
+        {
+            union.skipTo(nextKey);
+        }
+        finally
+        {
+            context.checkpoint();
+        }
+    }
+
+    public void close()
+    {
+        FileUtils.closeQuietly(union);
+        referencedIndexes.forEach(IndexSearchResultIterator::releaseQuietly);
+        referencedIndexes.clear();
+    }
+
+    private static void releaseQuietly(SSTableIndex index)
+    {
+        try
+        {
+            index.release();
+        }
+        catch (Throwable e)
+        {
+            logger.error(index.getIndexContext().logMessage(String.format("Failed to release index on SSTable %s", index.getSSTable())), e);
+        }
+    }
+}
diff --git a/test/unit/org/apache/cassandra/index/sai/memory/InMemoryKeyRangeIteratorTest.java b/src/java/org/apache/cassandra/index/sai/disk/PerColumnIndexWriter.java
similarity index 50%
copy from test/unit/org/apache/cassandra/index/sai/memory/InMemoryKeyRangeIteratorTest.java
copy to src/java/org/apache/cassandra/index/sai/disk/PerColumnIndexWriter.java
index 71db5c03d8..4ce75cbe08 100644
--- a/test/unit/org/apache/cassandra/index/sai/memory/InMemoryKeyRangeIteratorTest.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/PerColumnIndexWriter.java
@@ -15,24 +15,34 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.cassandra.index.sai.memory;
+package org.apache.cassandra.index.sai.disk;
 
-import java.util.Arrays;
-import java.util.SortedSet;
-import java.util.TreeSet;
+import java.io.IOException;
 
+import com.google.common.base.Stopwatch;
+
+import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.index.sai.utils.PrimaryKey;
-import org.apache.cassandra.index.sai.utils.KeyRangeIterator;
 
-public class InMemoryKeyRangeIteratorTest extends AbstractInMemoryKeyRangeIteratorTester
+/**
+ * Creates the on-disk components for a given column index.
+ */
+public interface PerColumnIndexWriter
 {
-    @Override
-    protected KeyRangeIterator makeIterator(long minimumTokenValue, long maximumTokenValue, long... tokens)
-    {
-        SortedSet<PrimaryKey> set = new TreeSet<>();
+    /**
+     * Adds a row to this index.
+     */
+    void addRow(PrimaryKey key, Row row, long sstableRowId) throws IOException;
 
-        Arrays.stream(tokens).forEach(t -> set.add(keyForToken(t)));
+    /**
+     * Builds on-disk index data structures from accumulated data, moves them all to the filesystem, and fsync created files.
+     */
+    void complete(Stopwatch stopwatch) throws IOException;
 
-        return new InMemoryKeyRangeIterator(set);
-    }
+    /**
+     * Aborts accumulating data. Allows to clean up resources on error.
+     * 
+     * Note: Implementations should be idempotent, i.e. safe to call multiple times without producing undesirable side-effects.
+     */
+    void abort(Throwable cause);
 }
diff --git a/src/java/org/apache/cassandra/index/sai/virtual/StorageAttachedIndexTables.java b/src/java/org/apache/cassandra/index/sai/disk/PerSSTableIndexWriter.java
similarity index 64%
copy from src/java/org/apache/cassandra/index/sai/virtual/StorageAttachedIndexTables.java
copy to src/java/org/apache/cassandra/index/sai/disk/PerSSTableIndexWriter.java
index 0d4f12678f..09c8a34a9e 100644
--- a/src/java/org/apache/cassandra/index/sai/virtual/StorageAttachedIndexTables.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/PerSSTableIndexWriter.java
@@ -15,21 +15,24 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+package org.apache.cassandra.index.sai.disk;
 
-package org.apache.cassandra.index.sai.virtual;
+import java.io.IOException;
 
-import java.util.Collection;
-import java.util.Collections;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
 
-import org.apache.cassandra.db.virtual.VirtualTable;
-
-public class StorageAttachedIndexTables
+/**
+ * Writes all SSTable-attached index files. The files written are version specific.
+ */
+public interface PerSSTableIndexWriter
 {
-    private StorageAttachedIndexTables()
+    PerSSTableIndexWriter NONE = (key) -> {};
+
+    void nextRow(PrimaryKey primaryKey) throws IOException;
+
+    default void complete() throws IOException
     {}
 
-    public static Collection<VirtualTable> getAll(String keyspace)
-    {
-        return Collections.singletonList(new IndexesSystemView(keyspace));
-    }
+    default void abort()
+    {}
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/PrimaryKeyMap.java b/src/java/org/apache/cassandra/index/sai/disk/PrimaryKeyMap.java
new file mode 100644
index 0000000000..fa47b9f4a4
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/PrimaryKeyMap.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+
+/**
+ * A bidirectional map of {@link PrimaryKey} to row ID. Implementations of this interface
+ * are not expected to be threadsafe.
+ */
+@NotThreadSafe
+public interface PrimaryKeyMap extends Closeable
+{
+    /**
+     * A factory for creating {@link PrimaryKeyMap} instances. Implementations of this
+     * interface are expected to be threadsafe.
+     */
+    @NotThreadSafe
+    interface Factory extends Closeable
+    {
+        /**
+         * Creates a new {@link PrimaryKeyMap} instance
+         *
+         * @return a {@link PrimaryKeyMap}
+         * @throws IOException if the {@link PrimaryKeyMap} couldn't be created
+         */
+        PrimaryKeyMap newPerSSTablePrimaryKeyMap() throws IOException;
+
+        @Override
+        default void close()
+        {
+        }
+    }
+
+    /**
+     * Returns a {@link PrimaryKey} for a row ID
+     *
+     * @param sstableRowId the row ID to lookup
+     * @return the {@link PrimaryKey} associated with the row ID
+     */
+    PrimaryKey primaryKeyFromRowId(long sstableRowId);
+
+    /**
+     * Returns a row ID for a {@link PrimaryKey}
+     *
+     * @param key the {@link PrimaryKey} to lookup
+     * @return the row ID associated with the {@link PrimaryKey}
+     */
+    long rowIdFromPrimaryKey(PrimaryKey key);
+
+    @Override
+    default void close()
+    {
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/RowMapping.java b/src/java/org/apache/cassandra/index/sai/disk/RowMapping.java
new file mode 100644
index 0000000000..fadbae5540
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/RowMapping.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk;
+
+import java.util.Collections;
+import java.util.Iterator;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+import com.carrotsearch.hppc.LongArrayList;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.tries.InMemoryTrie;
+import org.apache.cassandra.index.sai.memory.MemtableIndex;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.PrimaryKeys;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.utils.AbstractGuavaIterator;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+/**
+ * In memory representation of {@link PrimaryKey} to row ID mappings which only contains
+ * {@link Row} regardless of whether it's live or deleted. ({@link RangeTombstoneMarker} is not included.)
+ *
+ * While this inherits the threading behaviour of {@link InMemoryTrie} of single-writer / multiple-reader,
+ * since it is only used by {@link StorageAttachedIndexWriter}, which is not threadsafe, we can consider
+ * this class not threadsafe as well.
+ */
+@NotThreadSafe
+public class RowMapping
+{
+    private static final InMemoryTrie.UpsertTransformer<Long, Long> OVERWRITE_TRANSFORMER = (existing, update) -> update;
+
+    public static final RowMapping DUMMY = new RowMapping()
+    {
+        @Override
+        public Iterator<Pair<ByteComparable, LongArrayList>> merge(MemtableIndex index) { return Collections.emptyIterator(); }
+
+        @Override
+        public void complete() {}
+
+        @Override
+        public boolean isComplete()
+        {
+            return true;
+        }
+
+        @Override
+        public void add(PrimaryKey key, long sstableRowId) {}
+    };
+
+    private final InMemoryTrie<Long> rowMapping = new InMemoryTrie<>(BufferType.OFF_HEAP);
+
+    private boolean complete = false;
+
+    public PrimaryKey minKey;
+    public PrimaryKey maxKey;
+
+    public long maxSSTableRowId = -1;
+
+    private RowMapping()
+    {}
+
+    /**
+     * Create row mapping for FLUSH operation only.
+     */
+    public static RowMapping create(OperationType opType)
+    {
+        if (opType == OperationType.FLUSH)
+            return new RowMapping();
+        return DUMMY;
+    }
+
+    /**
+     * Link the term -> {@link PrimaryKeys} mappings from a provided {@link MemtableIndex} to
+     * the {@link PrimaryKey} -> row ID mappings maintained here in {@link #rowMapping} to produce
+     * mappings of terms to their postings lists.
+     *
+     * @param index a Memtable-attached column index
+     *
+     * @return an iterator of term -> postings list {@link Pair}s
+     */
+    public Iterator<Pair<ByteComparable, LongArrayList>> merge(MemtableIndex index)
+    {
+        assert complete : "RowMapping is not built.";
+
+        Iterator<Pair<ByteComparable, PrimaryKeys>> iterator = index.iterator();
+        return new AbstractGuavaIterator<Pair<ByteComparable, LongArrayList>>()
+        {
+            @Override
+            protected Pair<ByteComparable, LongArrayList> computeNext()
+            {
+                while (iterator.hasNext())
+                {
+                    Pair<ByteComparable, PrimaryKeys> pair = iterator.next();
+
+                    LongArrayList postings = null;
+                    Iterator<PrimaryKey> primaryKeys = pair.right.iterator();
+
+                    while (primaryKeys.hasNext())
+                    {
+                        PrimaryKey primaryKey = primaryKeys.next();
+                        ByteComparable byteComparable = primaryKey::asComparableBytes;
+                        Long sstableRowId = rowMapping.get(byteComparable);
+
+                        // The in-memory index does not handle deletions, so it is possible to
+                        // have a primary key in the index that doesn't exist in the row mapping
+                        if (sstableRowId != null)
+                        {
+                            postings = postings == null ? new LongArrayList() : postings;
+                            postings.add(sstableRowId);
+                        }
+                    }
+                    if (postings != null)
+                        return Pair.create(pair.left, postings);
+                }
+                return endOfData();
+            }
+        };
+    }
+
+    /**
+     * Complete building in memory RowMapping, mark it as immutable.
+     */
+    public void complete()
+    {
+        assert !complete : "RowMapping can only be built once.";
+        this.complete = true;
+    }
+
+    public boolean isComplete()
+    {
+        return complete;
+    }
+
+    /**
+     * Include PrimaryKey to RowId mapping
+     */
+    public void add(PrimaryKey key, long sstableRowId) throws InMemoryTrie.SpaceExhaustedException
+    {
+        assert !complete : "Cannot modify built RowMapping.";
+
+        ByteComparable byteComparable = key::asComparableBytes;
+        rowMapping.putSingleton(byteComparable, sstableRowId, OVERWRITE_TRANSFORMER);
+
+        maxSSTableRowId = Math.max(maxSSTableRowId, sstableRowId);
+
+        // data is written in token sorted order
+        if (minKey == null)
+            minKey = key;
+        maxKey = key;
+    }
+
+    public boolean hasRows()
+    {
+        return maxSSTableRowId >= 0;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/SSTableIndex.java b/src/java/org/apache/cassandra/index/sai/disk/SSTableIndex.java
new file mode 100644
index 0000000000..2227291626
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/SSTableIndex.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Objects;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.virtual.SimpleDataSet;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.io.sstable.SSTableIdFactory;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+/**
+ * A reference-counted container of a {@link SSTableReader} for each column index that:
+ * <ul>
+ *     <li>Manages references to the SSTable for each query</li>
+ *     <li>Exposes a version agnostic searcher onto the column index</li>
+ *     <li>Exposes the index metadata for the column index</li>
+ * </ul>
+ */
+public abstract class SSTableIndex
+{
+    private static final Logger logger = LoggerFactory.getLogger(SSTableIndex.class);
+
+    // sort sstable indexes by first key, then last key, then descriptor id
+    public static final Comparator<SSTableIndex> COMPARATOR = Comparator.comparing((SSTableIndex s) -> s.getSSTable().first)
+                                                                        .thenComparing(s -> s.getSSTable().last)
+                                                                        .thenComparing(s -> s.getSSTable().descriptor.id, SSTableIdFactory.COMPARATOR);
+
+    protected final SSTableContext sstableContext;
+    protected final IndexContext indexContext;
+
+    private final AtomicInteger references = new AtomicInteger(1);
+    private final AtomicBoolean obsolete = new AtomicBoolean(false);
+
+    public SSTableIndex(SSTableContext sstableContext, IndexContext indexContext)
+    {
+        assert indexContext.getValidator() != null;
+
+        this.sstableContext = sstableContext.sharedCopy(); // this line must not be before any code that may throw
+        this.indexContext = indexContext;
+    }
+
+    /**
+     * Returns the amount of memory occupied by the index when it is initially loaded.
+     * This is the amount of data loaded into internal memory buffers by the index and
+     * does include the class footprint overhead. It used by the index metrics.
+     */
+    public abstract long indexFileCacheSize();
+
+    /**
+     * Returns the number of indexed rows in the index. This comes from the index
+     * metadata created when the index was written and is used by the index metrics.
+     */
+    public abstract long getRowCount();
+
+    /**
+     * Returns the minimum indexed rowId for the index. This comes from the index
+     * metadata created when the index was written and is used by the index metrics.
+     */
+    public abstract long minSSTableRowId();
+
+    /**
+     * Returns the maximum indexed rowId for the index. This comes from the index
+     * metadata created when the index was written and is used by the index metrics.
+     */
+    public abstract long maxSSTableRowId();
+
+    /**
+     * Returns the minimum term held in the index based on the natural sort order of
+     * the index column type comparator. It comes from the index metadata created when
+     * the index was written and is used by the index metrics and used in queries to
+     * determine whether a term, or range or terms, exists in the index.
+     */
+    public abstract ByteBuffer minTerm();
+
+    /**
+     * Returns the maximum term held in the index based on the natural sort order of
+     * the index column type comparator. It comes from the index metadata created when
+     * the index was written and is used by the index metrics and used in queries to
+     * determine whether a term, or range or terms, exists in the index.
+     */
+    public abstract ByteBuffer maxTerm();
+
+    /**
+     * Returns the key bounds of the index. It is created from the minimum and
+     * maximum keys held in the metadata and is used to determine whether
+     * sstable indexes overlap or not.
+     */
+    public abstract AbstractBounds<PartitionPosition> bounds();
+
+    /**
+     * Perform a search on the index for a single expression and keyRange.
+     *
+     * The result is a {@link List} of {@link KeyRangeIterator} because there will
+     * be a {@link KeyRangeIterator} for each segment in the index. The result
+     * will never be null but may be an empty {@link List}.
+     *
+     * @param expression The {@link Expression} to be searched for
+     * @param keyRange The {@link AbstractBounds<PartitionPosition>} defining the
+     *                 token range for the search
+     * @param context The {@link QueryContext} holding the per-query state
+     * @return a {@link List} of {@link KeyRangeIterator}s containing the results
+     * of the search
+     */
+    public abstract List<KeyRangeIterator> search(Expression expression,
+                                                  AbstractBounds<PartitionPosition> keyRange,
+                                                  QueryContext context) throws IOException;
+
+    /**
+     * Populates a virtual table using the index metadata owned by the index
+     */
+    public abstract void populateSegmentView(SimpleDataSet dataSet);
+
+    protected abstract void internalRelease();
+
+    /**
+     * @return total size of per-column index components, in bytes
+     */
+    public long sizeOfPerColumnComponents()
+    {
+        return sstableContext.indexDescriptor.sizeOnDiskOfPerIndexComponents(indexContext);
+    }
+
+    public IndexContext getIndexContext()
+    {
+        return indexContext;
+    }
+
+    public SSTableContext getSSTableContext()
+    {
+        return sstableContext;
+    }
+
+    public Version getVersion()
+    {
+        return sstableContext.indexDescriptor.version;
+    }
+
+    public SSTableReader getSSTable()
+    {
+        return sstableContext.sstable;
+    }
+
+    public boolean reference()
+    {
+        while (true)
+        {
+            int n = references.get();
+            if (n <= 0)
+                return false;
+            if (references.compareAndSet(n, n + 1))
+            {
+                return true;
+            }
+        }
+    }
+
+    public boolean isReleased()
+    {
+        return references.get() <= 0;
+    }
+
+    public void releaseQuietly()
+    {
+        try
+        {
+            release();
+        }
+        catch (Throwable e)
+        {
+            logger.error(getIndexContext().logMessage("Failed to release index on SSTable {}"), getSSTable().descriptor, e);
+        }
+    }
+
+    public void release()
+    {
+        int n = references.decrementAndGet();
+
+        if (n == 0)
+        {
+            internalRelease();
+            sstableContext.close();
+
+            /*
+             * When SSTable is removed, storage-attached index components will be automatically removed by LogTransaction.
+             * We only remove index components explicitly in case of index corruption or index rebuild.
+             */
+            if (obsolete.get())
+            {
+                sstableContext.indexDescriptor.deleteColumnIndex(indexContext);
+            }
+        }
+    }
+
+    public void markObsolete()
+    {
+        obsolete.getAndSet(true);
+        release();
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        SSTableIndex other = (SSTableIndex)o;
+        return Objects.equal(sstableContext, other.sstableContext) && Objects.equal(indexContext, other.indexContext);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hashCode(sstableContext, indexContext);
+    }
+
+    @Override
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this)
+                          .add("column", indexContext.getColumnName())
+                          .add("sstable", sstableContext.sstable.descriptor)
+                          .add("minTerm", indexContext.getValidator().getString(minTerm()))
+                          .add("maxTerm", indexContext.getValidator().getString(maxTerm()))
+                          .add("totalRows", sstableContext.sstable.getTotalRows())
+                          .toString();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/StorageAttachedIndexWriter.java b/src/java/org/apache/cassandra/index/sai/disk/StorageAttachedIndexWriter.java
new file mode 100644
index 0000000000..f315e0dbba
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/StorageAttachedIndexWriter.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import com.google.common.base.Stopwatch;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.tries.InMemoryTrie;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.utils.Throwables;
+
+/**
+ * Writes all on-disk index structures attached to a given SSTable.
+ */
+@NotThreadSafe
+public class StorageAttachedIndexWriter implements SSTableFlushObserver
+{
+    private static final Logger logger = LoggerFactory.getLogger(StorageAttachedIndexWriter.class);
+
+    private final IndexDescriptor indexDescriptor;
+    private final Collection<StorageAttachedIndex> indexes;
+    private final Collection<PerColumnIndexWriter> perIndexWriters;
+    private final PerSSTableIndexWriter perSSTableWriter;
+    private final Stopwatch stopwatch = Stopwatch.createUnstarted();
+    private final RowMapping rowMapping;
+    private final boolean propagateErrors;
+
+    private DecoratedKey currentKey;
+    private boolean tokenOffsetWriterCompleted = false;
+    private boolean aborted = false;
+
+    private long sstableRowId = 0;
+
+    public static StorageAttachedIndexWriter createFlushObserverWriter(IndexDescriptor indexDescriptor,
+                                                                       Collection<StorageAttachedIndex> indexes,
+                                                                       LifecycleNewTracker lifecycleNewTracker) throws IOException
+    {
+        return new StorageAttachedIndexWriter(indexDescriptor, indexes, lifecycleNewTracker, false, false);
+
+    }
+
+    public static StorageAttachedIndexWriter createBuilderWriter(IndexDescriptor indexDescriptor,
+                                                                 Collection<StorageAttachedIndex> indexes,
+                                                                 LifecycleNewTracker lifecycleNewTracker,
+                                                                 boolean perIndexComponentsOnly) throws IOException
+    {
+        return new StorageAttachedIndexWriter(indexDescriptor, indexes, lifecycleNewTracker, perIndexComponentsOnly, true);
+    }
+
+    private StorageAttachedIndexWriter(IndexDescriptor indexDescriptor,
+                                       Collection<StorageAttachedIndex> indexes,
+                                       LifecycleNewTracker lifecycleNewTracker,
+                                       boolean perIndexComponentsOnly,
+                                       boolean propagateErrors) throws IOException
+    {
+        this.indexDescriptor = indexDescriptor;
+        this.indexes = indexes;
+        this.rowMapping = RowMapping.create(lifecycleNewTracker.opType());
+        this.propagateErrors = propagateErrors;
+        this.perIndexWriters = indexes.stream().map(index -> indexDescriptor.newPerColumnIndexWriter(index,
+                                                                                                     lifecycleNewTracker,
+                                                                                                     rowMapping))
+                                      .filter(Objects::nonNull) // a null here means the column had no data to flush
+                                      .collect(Collectors.toList());
+
+        // If the SSTable components are already being built by another index build then we don't want
+        // to build them again so use a null writer
+        this.perSSTableWriter = perIndexComponentsOnly ? PerSSTableIndexWriter.NONE : indexDescriptor.newPerSSTableIndexWriter();
+    }
+
+    @Override
+    public void begin()
+    {
+        logger.debug(indexDescriptor.logMessage("Starting partition iteration for storage-attached index flush for SSTable {}..."), indexDescriptor.sstableDescriptor);
+        stopwatch.start();
+    }
+
+    @Override
+    public void startPartition(DecoratedKey key, long keyPosition, long keyPositionForSASI)
+    {
+        if (aborted) return;
+        
+        currentKey = key;
+    }
+
+    @Override
+    public void nextUnfilteredCluster(Unfiltered unfiltered)
+    {
+        if (aborted) return;
+
+        // Ignore range tombstones...
+        if (!unfiltered.isRow())
+            return;
+
+        try
+        {
+            addRow((Row)unfiltered);
+        }
+        catch (Throwable t)
+        {
+            logger.error(indexDescriptor.logMessage("Failed to record a row during an index build"), t);
+            abort(t, true);
+        }
+    }
+
+    @Override
+    public void staticRow(Row staticRow)
+    {
+        if (aborted) return;
+        
+        if (staticRow.isEmpty())
+            return;
+
+        try
+        {
+            addRow(staticRow);
+        }
+        catch (Throwable t)
+        {
+            logger.error(indexDescriptor.logMessage("Failed to record a static row during an index build"), t);
+            abort(t, true);
+        }
+    }
+
+    @Override
+    public void complete()
+    {
+        if (aborted) return;
+
+        long start = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+
+        logger.debug(indexDescriptor.logMessage("Completed partition iteration for index flush for SSTable {}. Elapsed time: {} ms"),
+                     indexDescriptor.sstableDescriptor, start);
+
+        try
+        {
+            perSSTableWriter.complete();
+            tokenOffsetWriterCompleted = true;
+
+            long elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+
+            logger.debug(indexDescriptor.logMessage("Completed per-SSTable write for SSTable {}. Duration: {} ms. Total elapsed time: {} ms."),
+                         indexDescriptor.sstableDescriptor, elapsed - start, elapsed);
+
+            start = elapsed;
+
+            rowMapping.complete();
+
+            for (PerColumnIndexWriter perIndexWriter : perIndexWriters)
+            {
+                perIndexWriter.complete(stopwatch);
+            }
+            elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+            logger.debug(indexDescriptor.logMessage("Completed per-index writes for SSTable {}. Duration: {} ms. Total elapsed time: {} ms."),
+                         indexDescriptor.sstableDescriptor, elapsed - start, elapsed);
+        }
+        catch (Throwable t)
+        {
+            logger.error(indexDescriptor.logMessage("Failed to complete an index build"), t);
+            abort(t, true);
+        }
+    }
+
+    /**
+     * Aborts all column index writers and, only if they have not yet completed, SSTable-level component writers.
+     * 
+     * @param accumulator the initial exception thrown from the failed writer
+     */
+    @Override
+    public void abort(Throwable accumulator)
+    {
+        abort(accumulator, false);
+    }
+
+    /**
+     *
+     * @param accumulator original cause of the abort
+     * @param fromIndex true if the cause of the abort was the index itself, false otherwise
+     */
+    public void abort(Throwable accumulator, boolean fromIndex)
+    {
+        // Mark the write operation aborted, so we can short-circuit any further operations on the component writers.
+        aborted = true;
+        
+        // Make any indexes involved in this transaction non-queryable, as they will likely not match the backing table.
+        if (fromIndex)
+            indexes.forEach(StorageAttachedIndex::makeIndexNonQueryable);
+        
+        for (PerColumnIndexWriter perIndexWriter : perIndexWriters)
+        {
+            try
+            {
+                perIndexWriter.abort(accumulator);
+            }
+            catch (Throwable t)
+            {
+                if (accumulator != null)
+                {
+                    accumulator.addSuppressed(t);
+                }
+            }
+        }
+        
+        if (!tokenOffsetWriterCompleted)
+        {
+            // If the token/offset files have already been written successfully, they can be reused later. 
+            perSSTableWriter.abort();
+        }
+
+        // If the abort was from an index error and this is part of an index build operation then
+        // propagate the error up to the SecondaryIndexManager, so it can be correctly marked as failed.
+        if (fromIndex && propagateErrors)
+            throw Throwables.unchecked(accumulator);
+    }
+
+    private void addRow(Row row) throws IOException, InMemoryTrie.SpaceExhaustedException
+    {
+        PrimaryKey primaryKey = indexDescriptor.primaryKeyFactory.create(currentKey, row.clustering());
+        perSSTableWriter.nextRow(primaryKey);
+        rowMapping.add(primaryKey, sstableRowId);
+
+        for (PerColumnIndexWriter w : perIndexWriters)
+        {
+            w.addRow(primaryKey, row, sstableRowId);
+        }
+        sstableRowId++;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/format/IndexComponent.java b/src/java/org/apache/cassandra/index/sai/disk/format/IndexComponent.java
new file mode 100644
index 0000000000..670ba82c09
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/format/IndexComponent.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.format;
+
+import org.apache.cassandra.index.sai.disk.v1.postings.PostingsWriter;
+import org.apache.cassandra.index.sai.disk.v1.trie.TrieTermsDictionaryWriter;
+
+/**
+ * This is a definitive list of all the on-disk components for all versions
+ */
+public enum IndexComponent
+{
+    /**
+     * Metadata for per-column index components
+     */
+    META("Meta"),
+
+    /**
+     * Term dictionary written by {@link TrieTermsDictionaryWriter} stores mappings of term and
+     * file pointer to posting block on posting file.
+     */
+    TERMS_DATA("TermsData"),
+
+    /**
+     * Stores postings written by {@link PostingsWriter}
+     */
+    POSTING_LISTS("PostingLists"),
+
+    /**
+     * If present indicates that the column index build completed successfully
+     */
+    COLUMN_COMPLETION_MARKER("ColumnComplete"),
+
+
+    // per-sstable components
+    /**
+     * Partition key token value for rows including row tombstone and static row. (access key is rowId)
+     */
+    TOKEN_VALUES("TokenValues"),
+
+    /**
+     * An on-disk trie containing the primary keys used for looking up the rowId from a partition key
+     */
+    PRIMARY_KEY_TRIE("PrimaryKeyTrie"),
+
+    /**
+     * Prefix-compressed blocks of primary keys used for rowId to partition key lookups
+     */
+    PRIMARY_KEY_BLOCKS("PrimaryKeyBlocks"),
+
+    /**
+     * Encoded sequence of offsets to primary key blocks
+     */
+    PRIMARY_KEY_BLOCK_OFFSETS("PrimaryKeyBlockOffsets"),
+
+    /**
+     * Metadata for per-SSTable on-disk components.
+     */
+    GROUP_META("GroupMeta"),
+
+    /**
+     * If present indicates that the per-sstable index build completed successfully
+     */
+    GROUP_COMPLETION_MARKER("GroupComplete");
+
+    public final String name;
+
+    IndexComponent(String name)
+    {
+        this.name = name;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/format/IndexDescriptor.java b/src/java/org/apache/cassandra/index/sai/disk/format/IndexDescriptor.java
new file mode 100644
index 0000000000..7261671eca
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/format/IndexDescriptor.java
@@ -0,0 +1,433 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.format;
+
+import java.io.IOException;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Objects;
+import com.google.common.io.Files;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.disk.PerColumnIndexWriter;
+import org.apache.cassandra.index.sai.disk.PerSSTableIndexWriter;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.disk.RowMapping;
+import org.apache.cassandra.index.sai.disk.SSTableIndex;
+import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
+import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.IOUtils;
+
+/**
+ * The {@link IndexDescriptor} is an analog of the SSTable {@link Descriptor} and provides version
+ * specific information about the on-disk state of a {@link StorageAttachedIndex}.
+ *
+ * The {@link IndexDescriptor} is primarily responsible for maintaining a view of the on-disk state
+ * of an index for a specific {@link org.apache.cassandra.io.sstable.SSTable}.
+ *
+ * It is responsible for opening files for use by writers and readers.
+ *
+ * Its remaining responsibility is to act as a proxy to the {@link OnDiskFormat} associated with the
+ * index {@link Version}.
+ */
+public class IndexDescriptor
+{
+    private static final Logger logger = LoggerFactory.getLogger(IndexDescriptor.class);
+
+    public final Version version;
+    public final Descriptor sstableDescriptor;
+    public final ClusteringComparator clusteringComparator;
+    public final PrimaryKey.Factory primaryKeyFactory;
+
+    private IndexDescriptor(Version version, Descriptor sstableDescriptor, ClusteringComparator clusteringComparator)
+    {
+        this.version = version;
+        this.sstableDescriptor = sstableDescriptor;
+        this.clusteringComparator = clusteringComparator;
+        this.primaryKeyFactory = new PrimaryKey.Factory(clusteringComparator);
+    }
+
+    public static IndexDescriptor create(Descriptor descriptor, ClusteringComparator clusteringComparator)
+    {
+        return new IndexDescriptor(Version.LATEST, descriptor, clusteringComparator);
+    }
+
+    public static IndexDescriptor create(SSTableReader sstable)
+    {
+        for (Version version : Version.ALL)
+        {
+            IndexDescriptor indexDescriptor = new IndexDescriptor(version,
+                                                                  sstable.descriptor,
+                                                                  sstable.metadata().comparator);
+
+            if (version.onDiskFormat().isPerSSTableIndexBuildComplete(indexDescriptor))
+            {
+                return indexDescriptor;
+            }
+        }
+        return new IndexDescriptor(Version.LATEST,
+                                   sstable.descriptor,
+                                   sstable.metadata().comparator);
+    }
+
+    public String componentName(IndexComponent indexComponent)
+    {
+        return version.fileNameFormatter().format(indexComponent, null);
+    }
+
+    public PrimaryKeyMap.Factory newPrimaryKeyMapFactory(SSTableReader sstable)
+    {
+        return version.onDiskFormat().newPrimaryKeyMapFactory(this, sstable);
+    }
+
+    public SSTableIndex newSSTableIndex(SSTableContext sstableContext, IndexContext indexContext)
+    {
+        return version.onDiskFormat().newSSTableIndex(sstableContext, indexContext);
+    }
+
+    public PerSSTableIndexWriter newPerSSTableIndexWriter() throws IOException
+    {
+        return version.onDiskFormat().newPerSSTableIndexWriter(this);
+    }
+
+    public PerColumnIndexWriter newPerColumnIndexWriter(StorageAttachedIndex index,
+                                                        LifecycleNewTracker tracker,
+                                                        RowMapping rowMapping)
+    {
+        return version.onDiskFormat().newPerColumnIndexWriter(index, this, tracker, rowMapping);
+    }
+
+    public boolean isPerSSTableIndexBuildComplete()
+    {
+        return version.onDiskFormat().isPerSSTableIndexBuildComplete(this);
+    }
+
+    public boolean isPerColumnIndexBuildComplete(IndexContext indexContext)
+    {
+        return version.onDiskFormat().isPerColumnIndexBuildComplete(this, indexContext);
+    }
+
+    public boolean hasComponent(IndexComponent indexComponent)
+    {
+        return fileFor(indexComponent).exists();
+    }
+
+    public boolean hasComponent(IndexComponent indexComponent, IndexContext indexContext)
+    {
+        return fileFor(indexComponent, indexContext).exists();
+    }
+
+    public File fileFor(IndexComponent indexComponent)
+    {
+        return createFile(indexComponent, null);
+    }
+
+    public File fileFor(IndexComponent indexComponent, IndexContext indexContext)
+    {
+        return createFile(indexComponent, indexContext);
+    }
+
+    public boolean isIndexEmpty(IndexContext indexContext)
+    {
+        // The index is empty if the index build completed successfully in that both
+        // a GROUP_COMPLETION_MARKER companent and a COLUMN_COMPLETION_MARKER exist for
+        // the index and the number of per-index components is 1 indicating that only the
+        // COLUMN_COMPLETION_MARKER exists for the index, as this is the only file that
+        // will be written if the index is empty
+        return isPerColumnIndexBuildComplete(indexContext) && numberOfPerIndexComponents(indexContext) == 1;
+    }
+
+    @SuppressWarnings("UnstableApiUsage")
+    public void createComponentOnDisk(IndexComponent component) throws IOException
+    {
+        Files.touch(fileFor(component).toJavaIOFile());
+    }
+
+    @SuppressWarnings("UnstableApiUsage")
+    public void createComponentOnDisk(IndexComponent component, IndexContext indexContext) throws IOException
+    {
+        Files.touch(fileFor(component, indexContext).toJavaIOFile());
+    }
+
+    public IndexInput openPerSSTableInput(IndexComponent indexComponent)
+    {
+        File file = fileFor(indexComponent);
+        if (logger.isTraceEnabled())
+            logger.trace(logMessage("Opening blocking index input for file {} ({})"),
+                         file,
+                         FBUtilities.prettyPrintMemory(file.length()));
+
+        return IndexFileUtils.instance.openBlockingInput(file);
+    }
+
+    public IndexInput openPerIndexInput(IndexComponent indexComponent, IndexContext indexContext)
+    {
+        final File file = fileFor(indexComponent, indexContext);
+        if (logger.isTraceEnabled())
+            logger.trace(logMessage("Opening blocking index input for file {} ({})"),
+                         file,
+                         FBUtilities.prettyPrintMemory(file.length()));
+
+        return IndexFileUtils.instance.openBlockingInput(file);
+    }
+
+    public IndexOutputWriter openPerSSTableOutput(IndexComponent component) throws IOException
+    {
+        return openPerSSTableOutput(component, false);
+    }
+
+    public IndexOutputWriter openPerSSTableOutput(IndexComponent component, boolean append) throws IOException
+    {
+        final File file = fileFor(component);
+
+        if (logger.isTraceEnabled())
+            logger.trace(logMessage("Creating SSTable attached index output for component {} on file {}..."),
+                         component,
+                         file);
+
+        IndexOutputWriter writer = IndexFileUtils.instance.openOutput(file);
+
+        if (append)
+        {
+            writer.skipBytes(file.length());
+        }
+
+        return writer;
+    }
+
+    public IndexOutputWriter openPerIndexOutput(IndexComponent indexComponent, IndexContext indexContext) throws IOException
+    {
+        return openPerIndexOutput(indexComponent, indexContext, false);
+    }
+
+    public IndexOutputWriter openPerIndexOutput(IndexComponent component, IndexContext indexContext, boolean append) throws IOException
+    {
+        final File file = fileFor(component, indexContext);
+
+        if (logger.isTraceEnabled())
+            logger.trace(indexContext.logMessage("Creating sstable attached index output for component {} on file {}..."),
+                         component,
+                         file);
+
+        IndexOutputWriter writer = IndexFileUtils.instance.openOutput(file);
+
+        if (append)
+        {
+            writer.skipBytes(file.length());
+        }
+
+        return writer;
+    }
+
+    public FileHandle createPerSSTableFileHandle(IndexComponent indexComponent)
+    {
+        final File file = fileFor(indexComponent);
+
+        if (logger.isTraceEnabled())
+        {
+            logger.trace(logMessage("Opening {} file handle for {} ({})"),
+                         file, FBUtilities.prettyPrintMemory(file.length()));
+        }
+
+        return new FileHandle.Builder(file).mmapped(true).complete();
+    }
+
+    public FileHandle createPerIndexFileHandle(IndexComponent indexComponent, IndexContext indexContext)
+    {
+        final File file = fileFor(indexComponent, indexContext);
+
+        if (logger.isTraceEnabled())
+        {
+            logger.trace(indexContext.logMessage("Opening file handle for {} ({})"),
+                         file, FBUtilities.prettyPrintMemory(file.length()));
+        }
+
+        return new FileHandle.Builder(file).mmapped(true).complete();
+    }
+
+    public Set<Component> getLivePerSSTableComponents()
+    {
+        return version.onDiskFormat()
+                      .perSSTableIndexComponents()
+                      .stream()
+                      .filter(c -> fileFor(c).exists())
+                      .map(version::makePerSSTableComponent)
+                      .collect(Collectors.toSet());
+    }
+
+    public Set<Component> getLivePerIndexComponents(IndexContext indexContext)
+    {
+        return version.onDiskFormat()
+                      .perColumnIndexComponents(indexContext)
+                      .stream()
+                      .filter(c -> fileFor(c, indexContext).exists())
+                      .map(c -> version.makePerIndexComponent(c, indexContext))
+                      .collect(Collectors.toSet());
+    }
+
+    public long sizeOnDiskOfPerSSTableComponents()
+    {
+        return version.onDiskFormat()
+                      .perSSTableIndexComponents()
+                      .stream()
+                      .map(this::fileFor)
+                      .filter(File::exists)
+                      .mapToLong(File::length)
+                      .sum();
+    }
+
+    public long sizeOnDiskOfPerIndexComponents(IndexContext indexContext)
+    {
+        return version.onDiskFormat()
+                      .perColumnIndexComponents(indexContext)
+                      .stream()
+                      .map(c -> fileFor(c, indexContext))
+                      .filter(File::exists)
+                      .mapToLong(File::length)
+                      .sum();
+    }
+
+    @VisibleForTesting
+    public long sizeOnDiskOfPerIndexComponent(IndexComponent indexComponent, IndexContext indexContext)
+    {
+        File componentFile = fileFor(indexComponent, indexContext);
+        return componentFile.exists() ? componentFile.length() : 0;
+    }
+
+    @SuppressWarnings("BooleanMethodIsAlwaysInverted")
+    public boolean validatePerIndexComponents(IndexContext indexContext)
+    {
+        logger.info(indexContext.logMessage("Validating per-column index components"));
+        return version.onDiskFormat().validatePerColumnIndexComponents(this, indexContext, false);
+    }
+
+    @VisibleForTesting
+    public boolean validatePerIndexComponentsChecksum(IndexContext indexContext)
+    {
+        return version.onDiskFormat().validatePerColumnIndexComponents(this, indexContext, true);
+    }
+
+    @SuppressWarnings("BooleanMethodIsAlwaysInverted")
+    public boolean validatePerSSTableComponents()
+    {
+        return version.onDiskFormat().validatePerSSTableIndexComponents(this, false);
+    }
+
+    @SuppressWarnings("BooleanMethodIsAlwaysInverted")
+    public boolean validatePerSSTableComponentsChecksum()
+    {
+        return version.onDiskFormat().validatePerSSTableIndexComponents(this, true);
+    }
+
+    public void deletePerSSTableIndexComponents()
+    {
+        version.onDiskFormat()
+               .perSSTableIndexComponents()
+               .stream()
+               .map(this::fileFor)
+               .filter(File::exists)
+               .forEach(this::deleteComponent);
+    }
+
+    public void deleteColumnIndex(IndexContext indexContext)
+    {
+        version.onDiskFormat()
+               .perColumnIndexComponents(indexContext)
+               .stream()
+               .map(c -> fileFor(c, indexContext))
+               .filter(File::exists)
+               .forEach(this::deleteComponent);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hashCode(sstableDescriptor, version);
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        IndexDescriptor other = (IndexDescriptor)o;
+        return Objects.equal(sstableDescriptor, other.sstableDescriptor) &&
+               Objects.equal(version, other.version);
+    }
+
+    @Override
+    public String toString()
+    {
+        return sstableDescriptor.toString() + "-SAI";
+    }
+
+    public String logMessage(String message)
+    {
+        // Index names are unique only within a keyspace.
+        return String.format("[%s.%s.*] %s",
+                             sstableDescriptor.ksname,
+                             sstableDescriptor.cfname,
+                             message);
+    }
+
+    private File createFile(IndexComponent component, IndexContext indexContext)
+    {
+        Component customComponent = version.makePerIndexComponent(component, indexContext);
+        return sstableDescriptor.fileFor(customComponent);
+    }
+
+    private long numberOfPerIndexComponents(IndexContext indexContext)
+    {
+        return version.onDiskFormat()
+                      .perColumnIndexComponents(indexContext)
+                      .stream()
+                      .map(c -> fileFor(c, indexContext))
+                      .filter(File::exists)
+                      .count();
+    }
+
+    private void deleteComponent(File file)
+    {
+        logger.debug(logMessage("Deleting storage-attached index component file {}"), file);
+        try
+        {
+            IOUtils.deleteFilesIfExist(file.toPath());
+        }
+        catch (IOException e)
+        {
+            logger.warn(logMessage("Unable to delete storage-attached index component file {} due to {}."), file, e.getMessage(), e);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/format/OnDiskFormat.java b/src/java/org/apache/cassandra/index/sai/disk/format/OnDiskFormat.java
new file mode 100644
index 0000000000..cf19dceb6b
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/format/OnDiskFormat.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.format;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.disk.PerColumnIndexWriter;
+import org.apache.cassandra.index.sai.disk.PerSSTableIndexWriter;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.disk.RowMapping;
+import org.apache.cassandra.index.sai.disk.SSTableIndex;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+/**
+ * An interface to the on-disk format of an index. This provides format agnostic methods
+ * to read and write an on-disk format.
+ *
+ * The methods on this interface can be logically mapped into the following groups
+ * based on their method parameters:
+ * <ul>
+ *     <li>Methods taking no parameters. These methods return static information about the
+ *     format. This can include static information about the per-sstable components</li>
+ *     <li>Methods taking just an {@link IndexContext}. These methods return static information
+ *     specific to the index. This can be information relating to the type of index being used</li>
+ *     <li>Methods taking an {@link IndexDescriptor}. These methods interact with the on-disk components, or
+ *     return objects that will interact with the on-disk components, or return information about the on-disk
+ *     components. If they take an {@link IndexContext} as well they will be interacting with per-column index files;
+ *     otherwise they will be interacting with per-sstable index files</li>
+ *     <li>Methods taking an {@link IndexComponent}. These methods only interact with a single index component or
+ *     set of index components</li>
+ *
+ * </ul>
+ */
+public interface OnDiskFormat
+{
+    /**
+     * Returns a {@link PrimaryKeyMap.Factory} for the SSTable
+     *
+     * @param indexDescriptor The {@link IndexDescriptor} for the SSTable
+     * @param sstable The {@link SSTableReader} associated with the {@link IndexDescriptor}
+     */
+    PrimaryKeyMap.Factory newPrimaryKeyMapFactory(IndexDescriptor indexDescriptor, SSTableReader sstable);
+
+    /**
+     * Create a new {@link SSTableIndex} for an on-disk index.
+     *
+     * @param sstableContext The {@link SSTableContext} holding the per-SSTable information for the index
+     * @param indexContext The {@link IndexContext} holding the per-index information for the index
+     * @return the new {@link SSTableIndex} for the on-disk index
+     */
+    SSTableIndex newSSTableIndex(SSTableContext sstableContext, IndexContext indexContext);
+
+    /**
+     * Create a new {@link PerSSTableIndexWriter} to write the per-SSTable on-disk components of an index.
+     *
+     * @param indexDescriptor The {@link IndexDescriptor} for the SSTable
+     * @throws IOException if the writer couldn't be created
+     */
+    PerSSTableIndexWriter newPerSSTableIndexWriter(IndexDescriptor indexDescriptor) throws IOException;
+
+    /**
+     * Create a new {@link PerColumnIndexWriter} to write the per-column on-disk components of an index. The {@link LifecycleNewTracker}
+     * is used to determine the type of index write about to happen this will either be an
+     * {@code OperationType.FLUSH} indicating that we are about to flush a {@link org.apache.cassandra.index.sai.memory.MemtableIndex}
+     * or one of the other operation types indicating that we will be writing from an existing SSTable
+     *
+     * @param index The {@link StorageAttachedIndex} holding the current index build status
+     * @param indexDescriptor The {@link IndexDescriptor} for the SSTable
+     * @param tracker The {@link LifecycleNewTracker} for index build operation.
+     * @param rowMapping The {@link RowMapping} that is used to map rowID to {@code PrimaryKey} during the write operation
+     */
+    PerColumnIndexWriter newPerColumnIndexWriter(StorageAttachedIndex index,
+                                                 IndexDescriptor indexDescriptor,
+                                                 LifecycleNewTracker tracker,
+                                                 RowMapping rowMapping);
+
+    /**
+     * Returns true if the per-sstable index components have been built and are valid.
+     *
+     * @param indexDescriptor The {@link IndexDescriptor} for the SSTable SAI index
+     */
+    boolean isPerSSTableIndexBuildComplete(IndexDescriptor indexDescriptor);
+
+    /**
+     * Returns true if the per-column index components have been built and are valid.
+     *
+     * @param indexDescriptor The {@link IndexDescriptor} for the SSTable SAI index
+     * @param indexContext The {@link IndexContext} for the index
+     */
+    boolean isPerColumnIndexBuildComplete(IndexDescriptor indexDescriptor, IndexContext indexContext);
+
+    /**
+     * Validate all the per-SSTable on-disk components and throw if a component is not valid
+     *
+     * @param indexDescriptor The {@link IndexDescriptor} for the SSTable SAI index
+     * @param checksum {@code true} if the checksum should be tested as part of the validation
+     *
+     * @return true if all the per-SSTable components are valid
+     */
+    boolean validatePerSSTableIndexComponents(IndexDescriptor indexDescriptor, boolean checksum);
+
+    /**
+     * Validate all the per-column on-disk components and throw if a component is not valid
+     *
+     * @param indexDescriptor The {@link IndexDescriptor} for the SSTable SAI index
+     * @param indexContext The {@link IndexContext} holding the per-index information for the index
+     * @param checksum {@code true} if the checksum should be tested as part of the validation
+     *
+     * @return true if all the per-column components are valid
+     */
+    boolean validatePerColumnIndexComponents(IndexDescriptor indexDescriptor, IndexContext indexContext, boolean checksum);
+
+    /**
+     * Returns the set of {@link IndexComponent} for the per-SSTable part of an index.
+     * This is a complete set of components that could exist on-disk. It does not imply that the
+     * components currently exist on-disk.
+     */
+    Set<IndexComponent> perSSTableIndexComponents();
+
+    /**
+     * Returns the set of {@link IndexComponent} for the per-column part of an index.
+     * This is a complete set of components that could exist on-disk. It does not imply that the
+     * components currently exist on-disk.
+     *
+     * @param indexContext The {@link IndexContext} for the index
+     */
+    Set<IndexComponent> perColumnIndexComponents(IndexContext indexContext);
+
+    /**
+     * Return the number of open per-SSTable files that can be open during a query.
+     * This is a static indication of the files that can be held open by an index
+     * for queries. It is not a dynamic calculation.
+     */
+    int openFilesPerSSTableIndex();
+
+    /**
+     * Return the number of open per-column index files that can be open during a query.
+     * This is a static indication of the files that can be help open by an index
+     * for queries. It is not a dynamic calculation.
+     *
+     * @param indexContext The {@link IndexContext} for the index
+     */
+    int openFilesPerColumnIndex(IndexContext indexContext);
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/format/Version.java b/src/java/org/apache/cassandra/index/sai/disk/format/Version.java
new file mode 100644
index 0000000000..16e84d94c7
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/format/Version.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.format;
+
+import java.util.Comparator;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+import com.google.common.base.Objects;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.disk.v1.V1OnDiskFormat;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+
+/**
+ * Format version of indexing component, denoted as [major][minor]. Same forward-compatibility rules apply as to
+ * {@link org.apache.cassandra.io.sstable.format.Version}.
+ */
+public class Version implements Comparable<Version>
+{
+    private static final String SAI_DESCRIPTOR = "SAI";
+    private static final String SAI_SEPARATOR = "+";
+
+    // Current version
+    public static final Version AA = new Version("aa", V1OnDiskFormat.instance, (c, i) -> defaultFileNameFormat(c, i, "aa"));
+
+    // These should be added in reverse order so that the latest version is used first. Version matching tests
+    // are more likely to match the latest version, so we want to test that one first.
+    public static final SortedSet<Version> ALL = new TreeSet<Version>(Comparator.reverseOrder()) {{
+        add(AA);
+    }};
+
+    public static final Version EARLIEST = AA;
+    // The latest version can be configured to be an earlier version to support partial upgrades that don't
+    // write newer versions of the on-disk formats.
+    public static final Version LATEST = CassandraRelevantProperties.SAI_LATEST_VERSION.convert(Version::parse);
+
+    private final String version;
+    private final OnDiskFormat onDiskFormat;
+    private final FileNameFormatter fileNameFormatter;
+
+    private Version(String version, OnDiskFormat onDiskFormat, FileNameFormatter fileNameFormatter)
+    {
+        this.version = version;
+        this.onDiskFormat = onDiskFormat;
+        this.fileNameFormatter = fileNameFormatter;
+    }
+
+    public static Version parse(String versionString)
+    {
+        for (Version version : ALL)
+            if (version.version.equals(versionString))
+                return version;
+        throw new IllegalArgumentException("The version string " + versionString + " does not represent a valid SAI version. " +
+                                           "It should be one of " + ALL.stream().map(Version::toString).collect(Collectors.joining(", ")));
+    }
+
+    @Override
+    public int compareTo(Version other)
+    {
+        return version.compareTo(other.version);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hashCode(version);
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Version other = (Version)o;
+        return Objects.equal(version, other.version);
+    }
+
+    @Override
+    public String toString()
+    {
+        return version;
+    }
+
+    public boolean onOrAfter(Version other)
+    {
+        return version.compareTo(other.version) >= 0;
+    }
+
+    public OnDiskFormat onDiskFormat()
+    {
+        return onDiskFormat;
+    }
+
+    public Component makePerSSTableComponent(IndexComponent indexComponent)
+    {
+        return SSTableFormat.Components.Types.CUSTOM.createComponent(fileNameFormatter.format(indexComponent, null));
+    }
+
+    public Component makePerIndexComponent(IndexComponent indexComponent, IndexContext indexContext)
+    {
+        return SSTableFormat.Components.Types.CUSTOM.createComponent(fileNameFormatter.format(indexComponent, indexContext));
+    }
+
+    public FileNameFormatter fileNameFormatter()
+    {
+        return fileNameFormatter;
+    }
+
+    public interface FileNameFormatter
+    {
+        String format(IndexComponent indexComponent, IndexContext indexContext);
+    }
+
+    /**
+     * SAI default filename formatter. This is the current SAI on-disk filename format
+     *
+     * Format: <sstable descriptor>-SAI+<version>(+<index name>)+<component name>.db
+     * Note: The index name is excluded for per-SSTable index files that are shared
+     * across all the per-column indexes for the SSTable.
+     */
+    private static String defaultFileNameFormat(IndexComponent indexComponent, IndexContext indexContext, String version)
+    {
+        StringBuilder stringBuilder = new StringBuilder();
+
+        stringBuilder.append(SAI_DESCRIPTOR);
+        stringBuilder.append(SAI_SEPARATOR).append(version);
+        if (indexContext != null)
+            stringBuilder.append(SAI_SEPARATOR).append(indexContext.getIndexName());
+        stringBuilder.append(SAI_SEPARATOR).append(indexComponent.name);
+        stringBuilder.append(Descriptor.EXTENSION);
+
+        return stringBuilder.toString();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/io/IndexFileUtils.java b/src/java/org/apache/cassandra/index/sai/disk/io/IndexFileUtils.java
new file mode 100644
index 0000000000..1394d20555
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/io/IndexFileUtils.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.io;
+
+import java.io.IOException;
+import java.nio.ByteOrder;
+import java.util.zip.CRC32;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.compress.BufferType;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.io.util.SequentialWriterOption;
+import org.apache.lucene.store.IndexInput;
+
+public class IndexFileUtils
+{
+    @VisibleForTesting
+    protected static final SequentialWriterOption DEFAULT_WRITER_OPTION = SequentialWriterOption.newBuilder()
+                                                                                                .trickleFsync(DatabaseDescriptor.getTrickleFsync())
+                                                                                                .trickleFsyncByteInterval(DatabaseDescriptor.getTrickleFsyncIntervalInKiB() * 1024)
+                                                                                                .bufferType(BufferType.OFF_HEAP)
+                                                                                                .finishOnClose(true)
+                                                                                                .build();
+
+    public static final IndexFileUtils instance = new IndexFileUtils(DEFAULT_WRITER_OPTION);
+
+    private final SequentialWriterOption writerOption;
+
+    @VisibleForTesting
+    protected IndexFileUtils(SequentialWriterOption writerOption)
+    {
+        this.writerOption = writerOption;
+    }
+
+    @SuppressWarnings({"resource", "RedundantSuppression"})
+    public IndexOutputWriter openOutput(File file)
+    {
+        assert writerOption.finishOnClose() : "IndexOutputWriter relies on close() to sync with disk.";
+
+        return new IndexOutputWriter(new IncrementalChecksumSequentialWriter(file, writerOption));
+    }
+
+    public IndexInput openInput(FileHandle handle)
+    {
+        return IndexInputReader.create(handle);
+    }
+
+    @SuppressWarnings({"resource", "RedundantSuppression"})
+    public IndexInput openBlockingInput(File file)
+    {
+        FileHandle fileHandle = new FileHandle.Builder(file).complete();
+        RandomAccessReader randomReader = fileHandle.createReader();
+
+        return IndexInputReader.create(randomReader, fileHandle::close);
+    }
+
+    public interface ChecksumWriter
+    {
+        long getChecksum();
+    }
+
+    static class IncrementalChecksumSequentialWriter extends SequentialWriter implements ChecksumWriter
+    {
+        private final CRC32 checksum = new CRC32();
+
+        IncrementalChecksumSequentialWriter(File file, SequentialWriterOption writerOption)
+        {
+            super(file, writerOption);
+        }
+
+        @Override
+        public void writeByte(int b) throws IOException
+        {
+            super.writeByte(b);
+            checksum.update(b);
+        }
+
+        @Override
+        public void write(byte[] b) throws IOException
+        {
+            super.write(b);
+            checksum.update(b);
+        }
+
+        @Override
+        public void write(byte[] b, int off, int len) throws IOException
+        {
+            super.write(b, off, len);
+            checksum.update(b, off, len);
+        }
+
+        @Override
+        public void writeChar(int v) throws IOException
+        {
+            super.writeChar(v);
+            addTochecksum(v, 2);
+        }
+
+        @Override
+        public void writeInt(int v) throws IOException
+        {
+            super.writeInt(v);
+            addTochecksum(v, 4);
+        }
+
+        @Override
+        public void writeLong(long v) throws IOException
+        {
+            super.writeLong(v);
+            addTochecksum(v, 8);
+        }
+
+        public long getChecksum()
+        {
+            return checksum.getValue();
+        }
+
+        private void addTochecksum(long bytes, int count)
+        {
+            int origCount = count;
+            if (ByteOrder.BIG_ENDIAN == buffer.order())
+                while (count > 0) checksum.update((int) (bytes >>> (8 * --count)));
+            else
+                while (count > 0) checksum.update((int) (bytes >>> (8 * (origCount - count--))));
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/io/IndexInputReader.java b/src/java/org/apache/cassandra/index/sai/disk/io/IndexInputReader.java
new file mode 100644
index 0000000000..00a862e53a
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/io/IndexInputReader.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.io;
+
+
+import java.io.IOException;
+
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.IndexInput;
+
+/**
+ * This is a wrapper over a Cassandra {@link RandomAccessReader} that provides an {@link IndexInput}
+ * interface for Lucene classes that need {@link IndexInput}. This is an optimisation because the
+ * Lucene {@link DataInput} reads bytes one at a time whereas the {@link RandomAccessReader} is
+ * optimised to read multibyte objects faster.
+ */
+public class IndexInputReader extends IndexInput
+{
+    private final RandomAccessReader input;
+    private final Runnable doOnClose;
+
+    private IndexInputReader(RandomAccessReader input, Runnable doOnClose)
+    {
+        super(input.getPath());
+        this.input = input;
+        this.doOnClose = doOnClose;
+    }
+
+    public static IndexInputReader create(RandomAccessReader input)
+    {
+        return new IndexInputReader(input, () -> {});
+    }
+
+    public static IndexInputReader create(RandomAccessReader input, Runnable doOnClose)
+    {
+        return new IndexInputReader(input, doOnClose);
+    }
+
+    @SuppressWarnings({"resource", "RedundantSuppression"})
+    public static IndexInputReader create(FileHandle handle)
+    {
+        RandomAccessReader reader = handle.createReader();
+        return new IndexInputReader(reader, () -> {});
+    }
+
+    @Override
+    public byte readByte() throws IOException
+    {
+        return input.readByte();
+    }
+
+    @Override
+    public void readBytes(byte[] bytes, int off, int len) throws IOException
+    {
+        input.readFully(bytes, off, len);
+    }
+
+    /**
+     * Using {@link RandomAccessReader#readShort()} directly is faster than {@link DataInput#readShort()} which calls
+     * {@link DataInput#readByte()} one by one
+     */
+    @Override
+    public short readShort() throws IOException
+    {
+        return input.readShort();
+    }
+
+    /**
+     * Using {@link RandomAccessReader#readInt()} directly is faster than {@link DataInput#readInt()} which
+     * calls {@link DataInput#readByte()} one by one
+     */
+    @Override
+    public int readInt() throws IOException
+    {
+        return input.readInt();
+    }
+
+    /**
+     * Using {@link RandomAccessReader#readLong()} directly is faster than {@link DataInput#readLong()} which
+     * calls {@link DataInput#readByte()} one by one
+     */
+    @Override
+    public long readLong() throws IOException
+    {
+        return input.readLong();
+    }
+
+    @Override
+    public void close()
+    {
+        try
+        {
+            input.close();
+        }
+        finally
+        {
+            doOnClose.run();
+        }
+    }
+
+    @Override
+    public long getFilePointer()
+    {
+        return input.getFilePointer();
+    }
+
+    @Override
+    public void seek(long position)
+    {
+        input.seek(position);
+    }
+
+    @Override
+    public long length()
+    {
+        return input.length();
+    }
+
+    @Override
+    public IndexInput slice(String sliceDescription, long offset, long length)
+    {
+        throw new UnsupportedOperationException("Slice operations are not supported");
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/io/IndexOutputWriter.java b/src/java/org/apache/cassandra/index/sai/disk/io/IndexOutputWriter.java
new file mode 100644
index 0000000000..4208048cf6
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/io/IndexOutputWriter.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.io;
+
+import java.io.IOException;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import com.google.common.base.MoreObjects;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.lucene.store.IndexOutput;
+
+/**
+ * This is a wrapper over a Cassandra {@link SequentialWriter} that provides a Lucene {@link IndexOutput}
+ * interface for the Lucene index writers.
+ */
+@NotThreadSafe
+public class IndexOutputWriter extends IndexOutput
+{
+    private static final Logger logger = LoggerFactory.getLogger(IndexOutputWriter.class);
+
+    private final SequentialWriter out;
+    private boolean closed;
+
+    public IndexOutputWriter(SequentialWriter out)
+    {
+        super(out.getPath(), out.getPath());
+        this.out = out;
+    }
+
+    public void skipBytes(long length) throws IOException
+    {
+        this.out.skipBytes(length);
+    }
+
+    public File getFile()
+    {
+        return new File(out.getPath());
+    }
+
+    @Override
+    public long getChecksum()
+    {
+        return ((IndexFileUtils.ChecksumWriter)out).getChecksum();
+    }
+
+    @Override
+    public long getFilePointer()
+    {
+        return out.position();
+    }
+
+    @Override
+    public void writeBytes(byte[] bytes, int offset, int len) throws IOException
+    {
+        out.write(bytes, offset, len);
+    }
+
+    @Override
+    public void writeByte(byte b) throws IOException
+    {
+        out.writeByte(b);
+    }
+
+    @Override
+    public void close()
+    {
+        // IndexOutput#close contract allows any output to be closed multiple times,
+        // and Lucene does it in few places. SequentialWriter can be closed once.
+        if (!closed)
+        {
+            if (logger.isTraceEnabled())
+            {
+                logger.trace("Closing index output: {}", this);
+            }
+
+            // The writer should sync its contents to disk before closing...
+            out.close();
+            closed = true;
+        }
+    }
+
+    @Override
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this)
+                          .add("path", out.getPath())
+                          .add("bytesWritten", getFilePointer())
+                          .add("crc", getChecksum())
+                          .toString();
+    }
+
+    /**
+     * Returns {@link SequentialWriter} associated with this writer. Convenient when interacting with Cassandra codebase to
+     * write files to disk. Note that all bytes written to the returned writer will still contribute to the checksum.
+     *
+     * @return {@link SequentialWriter} associated with this writer
+     */
+    public SequentialWriter asSequentialWriter()
+    {
+        return out;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/io/RAMIndexOutput.java b/src/java/org/apache/cassandra/index/sai/disk/io/RAMIndexOutput.java
new file mode 100644
index 0000000000..709f4e6d40
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/io/RAMIndexOutput.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.io;
+
+import java.io.IOException;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.lucene.store.GrowableByteArrayDataOutput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * A replacement for {@link org.apache.lucene.store.RAMOutputStream}
+ * that doesn't use monitor locks.
+ */
+@NotThreadSafe
+public class RAMIndexOutput extends IndexOutput
+{
+    private final GrowableByteArrayDataOutput out;
+
+    public RAMIndexOutput(String name)
+    {
+        super("", name);
+        //TODO CASSANDRA-18280 to investigate the initial size allocation
+        out = new GrowableByteArrayDataOutput(128);
+    }
+
+    @Override
+    public long getChecksum()
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public long getFilePointer()
+    {
+        return out.getPosition();
+    }
+
+    @Override
+    public void writeByte(byte b)
+    {
+        out.writeByte(b);
+    }
+
+    @Override
+    public void writeBytes(byte[] bytes, int offset, int len)
+    {
+        out.writeBytes(bytes, offset, len);
+    }
+
+    public void writeTo(IndexOutput externalOut) throws IOException
+    {
+        externalOut.writeBytes(out.getBytes(), 0, out.getPosition());
+    }
+
+    public BytesRef getBytes()
+    {
+        return new BytesRef(out.getBytes(), 0, out.getPosition());
+    }
+
+    public void reset()
+    {
+        out.reset();
+    }
+
+    @Override
+    public void close()
+    {}
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/io/SeekingRandomAccessInput.java b/src/java/org/apache/cassandra/index/sai/disk/io/SeekingRandomAccessInput.java
new file mode 100644
index 0000000000..fcc5baed9d
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/io/SeekingRandomAccessInput.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.io;
+
+import java.io.IOException;
+
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.RandomAccessInput;
+
+/**
+ * {@link IndexInput} adapter that exposes it as a {@link RandomAccessInput}.
+ */
+public class SeekingRandomAccessInput implements RandomAccessInput
+{
+    private final IndexInput in;
+
+    public SeekingRandomAccessInput(IndexInput in)
+    {
+        this.in = in;
+    }
+
+    @Override
+    public byte readByte(long pos) throws IOException
+    {
+        in.seek(pos);
+        return in.readByte();
+    }
+
+    @Override
+    public short readShort(long pos) throws IOException
+    {
+        in.seek(pos);
+        return in.readShort();
+    }
+
+    @Override
+    public int readInt(long pos) throws IOException
+    {
+        in.seek(pos);
+        return in.readInt();
+    }
+
+    @Override
+    public long readLong(long pos) throws IOException
+    {
+        in.seek(pos);
+        return in.readLong();
+    }
+
+    @Override
+    public String toString()
+    {
+        return "SeekingRandomAccessInput(" + in + ')';
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/DirectReaders.java b/src/java/org/apache/cassandra/index/sai/disk/v1/DirectReaders.java
new file mode 100644
index 0000000000..5d5ebce539
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/DirectReaders.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.Sets;
+
+import org.apache.lucene.store.RandomAccessInput;
+
+public class DirectReaders
+{
+    public static final Set<Integer> SUPPORTED_BITS_PER_VALUE = Sets.newHashSet(0, 1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 64);
+    public static final String SUPPORTED_BITS_PER_VALUE_STRING = SUPPORTED_BITS_PER_VALUE.stream()
+                                                                                         .map(i -> Integer.toString(i))
+                                                                                         .collect(Collectors.joining(", "));
+
+    public interface Reader
+    {
+        /**
+         * Returns the indexed delta value from the block starting at offset.
+         * This is implemented for each of the supported bits-per-value where
+         * each implementation is responsible for decoding the bits.
+         */
+        long get(RandomAccessInput in, long offset, long index);
+    }
+
+    public static Reader getReaderForBitsPerValue(byte bitsPerValue)
+    {
+        switch (bitsPerValue)
+        {
+            case 0:
+                return READER_0;
+            case 1:
+                return READER_1;
+            case 2:
+                return READER_2;
+            case 4:
+                return READER_4;
+            case 8:
+                return READER_8;
+            case 12:
+                return READER_12;
+            case 16:
+                return READER_16;
+            case 20:
+                return READER_20;
+            case 24:
+                return READER_24;
+            case 28:
+                return READER_28;
+            case 32:
+                return READER_32;
+            case 40:
+                return READER_40;
+            case 48:
+                return READER_48;
+            case 56:
+                return READER_56;
+            case 64:
+                return READER_64;
+            default:
+                throw new IllegalArgumentException("unsupported bitsPerValue: " + bitsPerValue);
+        }
+    }
+
+    private static final Reader READER_0 = (in, offset, index) -> 0;
+
+    private static final Reader READER_1 = (in, offset, index) -> {
+        try
+        {
+            int shift = 7 - (int) (index & 7);
+            return (in.readByte(offset + (index >>> 3)) >>> shift) & 0x1;
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    };
+
+    private static final Reader READER_2 = (in, offset, index) -> {
+        try
+        {
+            int shift = (3 - (int) (index & 3)) << 1;
+            return (in.readByte(offset + (index >>> 2)) >>> shift) & 0x3;
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    };
+
+    private static final Reader READER_4 = (in, offset, index) -> {
+        try
+        {
+            int shift = (int) ((index + 1) & 1) << 2;
+            return (in.readByte(offset + (index >>> 1)) >>> shift) & 0xF;
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    };
+
+    private static final Reader READER_8 = (in, offset, index) -> {
+        try
+        {
+            return in.readByte(offset + index) & 0xFF;
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    };
+
+    private static final Reader READER_12 = (in, offset, index) -> {
+        try
+        {
+            long o = (index * 12) >>> 3;
+            int shift = (int) ((index + 1) & 1) << 2;
+            return (in.readShort(offset + o) >>> shift) & 0xFFF;
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    };
+
+    private static final Reader READER_16 = (in, offset, index) -> {
+        try
+        {
+            return in.readShort(offset + (index << 1)) & 0xFFFF;
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    };
+
+    private static final Reader READER_20 = (in, offset, index) -> {
+        try
+        {
+            long o = (index * 20) >>> 3;
+            int v = in.readInt(offset + o) >>> 8;
+            int shift = (int) ((index + 1) & 1) << 2;
+            return (v >>> shift) & 0xFFFFF;
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    };
+
+    private static final Reader READER_24 = (in, offset, index) -> {
+        try
+        {
+            return in.readInt(offset + index * 3) >>> 8;
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    };
+
+    private static final Reader READER_28 = (in, offset, index) -> {
+        try
+        {
+            long o = (index * 28) >>> 3;
+            int shift = (int) ((index + 1) & 1) << 2;
+            return (in.readInt(offset + o) >>> shift) & 0xFFFFFFFL;
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    };
+
+    private static final Reader READER_32 = (in, offset, index) -> {
+        try
+        {
+            return in.readInt(offset + (index << 2)) & 0xFFFFFFFFL;
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    };
+
+    private static final Reader READER_40 = (in, offset, index) -> {
+        try
+        {
+            return in.readLong(offset + index * 5) >>> 24;
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    };
+
+    private static final Reader READER_48 = (in, offset, index) -> {
+        try
+        {
+            return in.readLong(offset + index * 6) >>> 16;
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    };
+
+    private static final Reader READER_56 = (in, offset, index) -> {
+        try
+        {
+            return in.readLong(offset + index * 7) >>> 8;
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    };
+
+    private static final Reader READER_64 = (in, offset, index) -> {
+        try
+        {
+            return in.readLong(offset + (index << 3));
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+    };
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/LongArray.java b/src/java/org/apache/cassandra/index/sai/disk/v1/LongArray.java
new file mode 100644
index 0000000000..2c2a1fc39b
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/LongArray.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.function.Supplier;
+import javax.annotation.concurrent.NotThreadSafe;
+
+/**
+ * Abstraction over a long-indexed array of longs.
+ */
+public interface LongArray extends Closeable
+{
+    /**
+     * Get value at {@code idx}.
+     */
+    long get(long idx);
+
+    /**
+     * Get array length.
+     */
+    long length();
+
+    @Override
+    default void close() throws IOException { }
+
+    @NotThreadSafe
+    class DeferredLongArray implements LongArray
+    {
+        private final Supplier<LongArray> supplier;
+        private LongArray longArray;
+        private boolean opened = false;
+
+        public DeferredLongArray(Supplier<LongArray> supplier)
+        {
+            this.supplier = supplier;
+        }
+
+        @Override
+        public long get(long idx)
+        {
+            open();
+            return longArray.get(idx);
+        }
+
+        @Override
+        public long length()
+        {
+            open();
+            return longArray.length();
+        }
+
+        @Override
+        public void close() throws IOException
+        {
+            if (opened)
+                longArray.close();
+        }
+
+        private void open()
+        {
+            if (!opened)
+            {
+                longArray = supplier.get();
+                opened = true;
+            }
+        }
+    }
+
+    interface Factory
+    {
+        LongArray open();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/MemtableIndexWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/MemtableIndexWriter.java
new file mode 100644
index 0000000000..6a497fbb33
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/MemtableIndexWriter.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.Stopwatch;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.carrotsearch.hppc.LongArrayList;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.disk.PerColumnIndexWriter;
+import org.apache.cassandra.index.sai.disk.RowMapping;
+import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
+import org.apache.cassandra.index.sai.disk.v1.trie.LiteralIndexWriter;
+import org.apache.cassandra.index.sai.memory.MemtableIndex;
+import org.apache.cassandra.index.sai.memory.MemtableTermsIterator;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.utils.Pair;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+/**
+ * Column index writer that flushes indexed data directly from the corresponding Memtable index, without buffering index
+ * data in memory.
+ */
+public class MemtableIndexWriter implements PerColumnIndexWriter
+{
+    private static final Logger logger = LoggerFactory.getLogger(MemtableIndexWriter.class);
+
+    private final IndexDescriptor indexDescriptor;
+    private final IndexContext indexContext;
+    private final MemtableIndex memtable;
+    private final RowMapping rowMapping;
+
+    public MemtableIndexWriter(MemtableIndex memtable,
+                               IndexDescriptor indexDescriptor,
+                               IndexContext indexContext,
+                               RowMapping rowMapping)
+    {
+        assert rowMapping != null && rowMapping != RowMapping.DUMMY : "Row mapping must exist during FLUSH.";
+
+        this.indexDescriptor = indexDescriptor;
+        this.indexContext = indexContext;
+        this.memtable = memtable;
+        this.rowMapping = rowMapping;
+    }
+
+    @Override
+    public void addRow(PrimaryKey key, Row row, long sstableRowId)
+    {
+        // Memtable indexes are flushed directly to disk with the aid of a mapping between primary
+        // keys and row IDs in the flushing SSTable. This writer, therefore, does nothing in
+        // response to the flushing of individual rows.
+    }
+
+    @Override
+    public void abort(Throwable cause)
+    {
+        logger.warn(indexContext.logMessage("Aborting index memtable flush for {}..."), indexDescriptor.sstableDescriptor, cause);
+        indexDescriptor.deleteColumnIndex(indexContext);
+    }
+
+    @Override
+    public void complete(Stopwatch stopwatch) throws IOException
+    {
+        assert rowMapping.isComplete() : "Cannot complete the memtable index writer because the row mapping is not complete";
+
+        long start = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+
+        try
+        {
+            if (!rowMapping.hasRows() || memtable == null || memtable.isEmpty())
+            {
+                logger.debug(indexContext.logMessage("No indexed rows to flush from SSTable {}."), indexDescriptor.sstableDescriptor);
+                // Write a completion marker even though we haven't written anything to the index,
+                // so we won't try to build the index again for the SSTable
+                indexDescriptor.createComponentOnDisk(IndexComponent.COLUMN_COMPLETION_MARKER, indexContext);
+                return;
+            }
+
+            final DecoratedKey minKey = rowMapping.minKey.partitionKey();
+            final DecoratedKey maxKey = rowMapping.maxKey.partitionKey();
+
+            final Iterator<Pair<ByteComparable, LongArrayList>> iterator = rowMapping.merge(memtable);
+
+            try (MemtableTermsIterator terms = new MemtableTermsIterator(memtable.getMinTerm(), memtable.getMaxTerm(), iterator))
+            {
+                long cellCount = flush(minKey, maxKey, indexContext.getValidator(), terms);
+
+                indexDescriptor.createComponentOnDisk(IndexComponent.COLUMN_COMPLETION_MARKER, indexContext);
+
+                indexContext.getIndexMetrics().memtableIndexFlushCount.inc();
+
+                long elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+
+                logger.debug(indexContext.logMessage("Completed flushing {} memtable index cells to SSTable {}. Duration: {} ms. Total elapsed: {} ms"),
+                             cellCount, indexDescriptor.sstableDescriptor, elapsed - start, elapsed);
+
+                indexContext.getIndexMetrics().memtableFlushCellsPerSecond.update((long) (cellCount * 1000.0 / Math.max(1, elapsed - start)));
+            }
+        }
+        catch (Throwable t)
+        {
+            logger.error(indexContext.logMessage("Error while flushing index {}"), t.getMessage(), t);
+            indexContext.getIndexMetrics().memtableIndexFlushErrors.inc();
+
+            throw t;
+        }
+    }
+
+    private long flush(DecoratedKey minKey,
+                       DecoratedKey maxKey,
+                       AbstractType<?> termComparator,
+                       MemtableTermsIterator terms) throws IOException
+    {
+        long numRows = 0;
+        SegmentMetadata.ComponentMetadataMap indexMetas = null;
+
+        if (TypeUtil.isLiteral(termComparator))
+        {
+            try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexContext))
+            {
+                indexMetas = writer.writeCompleteSegment(terms);
+                numRows = writer.getPostingsCount();
+            }
+        }
+
+        // If no rows were written we need to delete any created column index components
+        // so that the index is correctly identified as being empty (only having a completion marker)
+        if (numRows == 0)
+        {
+            indexDescriptor.deleteColumnIndex(indexContext);
+            return 0;
+        }
+
+        // During index memtable flush, the data is sorted based on terms.
+        SegmentMetadata metadata = new SegmentMetadata(0,
+                                                       numRows,
+                                                       terms.getMinSSTableRowId(),
+                                                       terms.getMaxSSTableRowId(),
+                                                       indexDescriptor.primaryKeyFactory.createPartitionKeyOnly(minKey),
+                                                       indexDescriptor.primaryKeyFactory.createPartitionKeyOnly(maxKey),
+                                                       terms.getMinTerm(),
+                                                       terms.getMaxTerm(),
+                                                       indexMetas);
+
+        try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, indexContext)))
+        {
+            SegmentMetadata.write(writer, Collections.singletonList(metadata));
+        }
+
+        return numRows;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/MetadataSource.java b/src/java/org/apache/cassandra/index/sai/disk/v1/MetadataSource.java
new file mode 100644
index 0000000000..b8b7fb66cc
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/MetadataSource.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.lucene.store.BufferedChecksumIndexInput;
+import org.apache.lucene.store.ByteArrayIndexInput;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.BytesRef;
+
+@NotThreadSafe
+public class MetadataSource
+{
+    private final Map<String, BytesRef> components;
+
+    private MetadataSource(Map<String, BytesRef> components)
+    {
+        this.components = components;
+    }
+
+    public static MetadataSource loadGroupMetadata(IndexDescriptor indexDescriptor) throws IOException
+    {
+        return MetadataSource.load(indexDescriptor.openPerSSTableInput(IndexComponent.GROUP_META));
+    }
+
+    public static MetadataSource loadColumnMetadata(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException
+    {
+        return MetadataSource.load(indexDescriptor.openPerIndexInput(IndexComponent.META, indexContext));
+    }
+
+    private static MetadataSource load(IndexInput indexInput) throws IOException
+    {
+        Map<String, BytesRef> components = new HashMap<>();
+
+        try (BufferedChecksumIndexInput input = new BufferedChecksumIndexInput(indexInput))
+        {
+            SAICodecUtils.checkHeader(input);
+            final int num = input.readInt();
+
+            for (int x = 0; x < num; x++)
+            {
+                if (input.length() == input.getFilePointer())
+                {
+                    // we should never get here, because we always add footer to the file
+                    throw new IllegalStateException("Unexpected EOF in " + input);
+                }
+
+                final String name = input.readString();
+                final int length = input.readInt();
+                final byte[] bytes = new byte[length];
+                input.readBytes(bytes, 0, length);
+
+                components.put(name, new BytesRef(bytes));
+            }
+
+            SAICodecUtils.checkFooter(input);
+        }
+
+        return new MetadataSource(components);
+    }
+
+    public IndexInput get(String name)
+    {
+        BytesRef bytes = components.get(name);
+
+        if (bytes == null)
+        {
+            throw new IllegalArgumentException(String.format("Could not find component '%s'. Available properties are %s.",
+                                                             name, components.keySet()));
+        }
+
+        return new ByteArrayIndexInput(name, bytes.bytes);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/MetadataWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/MetadataWriter.java
new file mode 100644
index 0000000000..b995f4ef17
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/MetadataWriter.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.index.sai.disk.io.RAMIndexOutput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.BytesRef;
+
+@NotThreadSafe
+public class MetadataWriter implements Closeable
+{
+    private final IndexOutput output;
+    private final Map<String, BytesRef> map = new HashMap<>();
+
+    public MetadataWriter(IndexOutput output)
+    {
+        this.output = output;
+    }
+
+    public Builder builder(String name)
+    {
+        return new Builder(name);
+    }
+
+    public class Builder extends RAMIndexOutput implements Closeable
+    {
+        private Builder(String name)
+        {
+            super(name);
+        }
+
+        @Override
+        public void close()
+        {
+            map.put(getName(), getBytes());
+        }
+    }
+
+    private void finish() throws IOException
+    {
+        SAICodecUtils.writeHeader(output);
+        output.writeInt(map.size());
+        for (Map.Entry<String, BytesRef> entry : map.entrySet())
+        {
+            output.writeString(entry.getKey());
+            output.writeInt(entry.getValue().length);
+            output.writeBytes(entry.getValue().bytes, entry.getValue().offset, entry.getValue().length);
+        }
+        SAICodecUtils.writeFooter(output);
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        try
+        {
+            finish();
+        }
+        finally
+        {
+            output.close();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/PerColumnIndexFiles.java b/src/java/org/apache/cassandra/index/sai/disk/v1/PerColumnIndexFiles.java
new file mode 100644
index 0000000000..7d129ba28f
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/PerColumnIndexFiles.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.Closeable;
+import java.util.EnumMap;
+import java.util.Map;
+
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+
+/**
+ * Maintains a mapping of {@link IndexComponent}s to associated {@link FileHandle}s for
+ * read operations on the components. Users of this class are returned copies of the
+ * {@link FileHandle}s using {@link FileHandle#sharedCopy()} so returned handles still
+ * need to be closed by the user.
+ */
+public class PerColumnIndexFiles implements Closeable
+{
+    private final Map<IndexComponent, FileHandle> files = new EnumMap<>(IndexComponent.class);
+    private final IndexDescriptor indexDescriptor;
+    private final IndexContext indexContext;
+
+    public PerColumnIndexFiles(IndexDescriptor indexDescriptor, IndexContext indexContext)
+    {
+        this.indexDescriptor = indexDescriptor;
+        this.indexContext = indexContext;
+        if (TypeUtil.isLiteral(indexContext.getValidator()))
+        {
+            files.put(IndexComponent.POSTING_LISTS, indexDescriptor.createPerIndexFileHandle(IndexComponent.POSTING_LISTS, indexContext));
+            files.put(IndexComponent.TERMS_DATA, indexDescriptor.createPerIndexFileHandle(IndexComponent.TERMS_DATA, indexContext));
+        }
+    }
+
+    public FileHandle termsData()
+    {
+        return getFile(IndexComponent.TERMS_DATA);
+    }
+
+    public FileHandle postingLists()
+    {
+        return getFile(IndexComponent.POSTING_LISTS);
+    }
+
+    @SuppressWarnings({"resource", "RedundantSuppression"})
+    private FileHandle getFile(IndexComponent indexComponent)
+    {
+        FileHandle file = files.get(indexComponent);
+        if (file == null)
+            throw new IllegalArgumentException(String.format(indexContext.logMessage("Component %s not found for SSTable %s"),
+                                                             indexComponent, indexDescriptor.sstableDescriptor));
+
+        return file.sharedCopy();
+    }
+
+    @Override
+    public void close()
+    {
+        FileUtils.closeQuietly(files.values());
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/RowAwarePrimaryKeyMap.java b/src/java/org/apache/cassandra/index/sai/disk/v1/RowAwarePrimaryKeyMap.java
new file mode 100644
index 0000000000..d06969f21a
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/RowAwarePrimaryKeyMap.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import javax.annotation.concurrent.NotThreadSafe;
+import javax.annotation.concurrent.ThreadSafe;
+
+import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringComparator;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.v1.bitpack.BlockPackedReader;
+import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesMeta;
+import org.apache.cassandra.index.sai.disk.v1.sortedterms.SortedTermsMeta;
+import org.apache.cassandra.index.sai.disk.v1.sortedterms.SortedTermsReader;
+import org.apache.cassandra.index.sai.disk.v1.trie.TriePrefixSearcher;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse;
+
+/**
+ * A row-aware {@link PrimaryKeyMap}
+ *
+ * This uses the following on-disk structures:
+ * <ul>
+ *     <li>Block-packed structure for rowId to token lookups using {@link BlockPackedReader}.
+ *     Uses component {@link IndexComponent#TOKEN_VALUES} </li>
+ *     <li>A sorted-terms structure for rowId to {@link PrimaryKey} and {@link PrimaryKey} to rowId lookups using
+ *     {@link SortedTermsReader} and {@link TriePrefixSearcher}. Uses components {@link IndexComponent#PRIMARY_KEY_TRIE},
+ *     {@link IndexComponent#PRIMARY_KEY_BLOCKS} and {@link IndexComponent#PRIMARY_KEY_BLOCK_OFFSETS}</li>
+ * </ul>
+ *
+ * While the {@link RowAwarePrimaryKeyMapFactory} is threadsafe, individual instances of the {@link RowAwarePrimaryKeyMap}
+ * are not.
+ */
+@NotThreadSafe
+public class RowAwarePrimaryKeyMap implements PrimaryKeyMap
+{
+    @ThreadSafe
+    public static class RowAwarePrimaryKeyMapFactory implements Factory
+    {
+        private final LongArray.Factory tokenReaderFactory;
+        private final SortedTermsReader sortedTermsReader;
+        private final SortedTermsMeta sortedTermsMeta;
+        private FileHandle tokensFile = null;
+        private FileHandle primaryKeyBlockOffsetsFile = null;
+        private FileHandle primaryKeyBlocksFile = null;
+        private FileHandle primaryKeyTrieFile = null;
+        private final IPartitioner partitioner;
+        private final ClusteringComparator clusteringComparator;
+        private final PrimaryKey.Factory primaryKeyFactory;
+
+        public RowAwarePrimaryKeyMapFactory(IndexDescriptor indexDescriptor, SSTableReader sstable)
+        {
+            try
+            {
+                MetadataSource metadataSource = MetadataSource.loadGroupMetadata(indexDescriptor);
+                NumericValuesMeta tokensMeta = new NumericValuesMeta(metadataSource.get(indexDescriptor.componentName(IndexComponent.TOKEN_VALUES)));
+                NumericValuesMeta blockOffsetsMeta = new NumericValuesMeta(metadataSource.get(indexDescriptor.componentName(IndexComponent.PRIMARY_KEY_BLOCK_OFFSETS)));
+                this.sortedTermsMeta = new SortedTermsMeta(metadataSource.get(indexDescriptor.componentName(IndexComponent.PRIMARY_KEY_BLOCKS)));
+                this.tokensFile = indexDescriptor.createPerSSTableFileHandle(IndexComponent.TOKEN_VALUES);
+                this.tokenReaderFactory = new BlockPackedReader(tokensFile, tokensMeta);
+                this.primaryKeyBlockOffsetsFile = indexDescriptor.createPerSSTableFileHandle(IndexComponent.PRIMARY_KEY_BLOCK_OFFSETS);
+                this.primaryKeyBlocksFile = indexDescriptor.createPerSSTableFileHandle(IndexComponent.PRIMARY_KEY_BLOCKS);
+                this.primaryKeyTrieFile = indexDescriptor.createPerSSTableFileHandle(IndexComponent.PRIMARY_KEY_TRIE);
+                this.sortedTermsReader = new SortedTermsReader(primaryKeyBlocksFile, primaryKeyBlockOffsetsFile, sortedTermsMeta, blockOffsetsMeta);
+                this.partitioner = sstable.metadata().partitioner;
+                this.primaryKeyFactory = indexDescriptor.primaryKeyFactory;
+                this.clusteringComparator = indexDescriptor.clusteringComparator;
+            }
+            catch (Throwable t)
+            {
+                throw Throwables.unchecked(Throwables.close(t, Arrays.asList(tokensFile, primaryKeyBlocksFile, primaryKeyBlockOffsetsFile, primaryKeyTrieFile)));
+            }
+        }
+
+        @Override
+        @SuppressWarnings({"resource", "RedundantSuppression"})
+        public PrimaryKeyMap newPerSSTablePrimaryKeyMap() throws IOException
+        {
+            LongArray rowIdToToken = new LongArray.DeferredLongArray(tokenReaderFactory::open);
+            return new RowAwarePrimaryKeyMap(rowIdToToken,
+                                             new TriePrefixSearcher(primaryKeyTrieFile.instantiateRebufferer(null), sortedTermsMeta.trieFilePointer),
+                                             sortedTermsReader.openCursor(),
+                                             partitioner,
+                                             primaryKeyFactory,
+                                             clusteringComparator);
+        }
+
+        @Override
+        public void close()
+        {
+            FileUtils.closeQuietly(Arrays.asList(tokensFile, primaryKeyBlocksFile, primaryKeyBlockOffsetsFile, primaryKeyTrieFile));
+        }
+    }
+
+    private final LongArray rowIdToToken;
+    private final TriePrefixSearcher triePrefixSearcher;
+    private final SortedTermsReader.Cursor cursor;
+    private final IPartitioner partitioner;
+    private final PrimaryKey.Factory primaryKeyFactory;
+    private final ClusteringComparator clusteringComparator;
+    private final ByteBuffer tokenBuffer = ByteBuffer.allocate(Long.BYTES);
+
+    private RowAwarePrimaryKeyMap(LongArray rowIdToToken,
+                                  TriePrefixSearcher triePrefixSearcher,
+                                  SortedTermsReader.Cursor cursor,
+                                  IPartitioner partitioner,
+                                  PrimaryKey.Factory primaryKeyFactory,
+                                  ClusteringComparator clusteringComparator)
+    {
+        this.rowIdToToken = rowIdToToken;
+        this.triePrefixSearcher = triePrefixSearcher;
+        this.cursor = cursor;
+        this.partitioner = partitioner;
+        this.primaryKeyFactory = primaryKeyFactory;
+        this.clusteringComparator = clusteringComparator;
+    }
+
+    @Override
+    public PrimaryKey primaryKeyFromRowId(long sstableRowId)
+    {
+        tokenBuffer.putLong(rowIdToToken.get(sstableRowId));
+        tokenBuffer.rewind();
+        return primaryKeyFactory.createDeferred(partitioner.getTokenFactory().fromByteArray(tokenBuffer), () -> supplier(sstableRowId));
+    }
+
+    @Override
+    public long rowIdFromPrimaryKey(PrimaryKey key)
+    {
+        return triePrefixSearcher.prefixSearch(key.asComparableBytes(ByteComparable.Version.OSS42));
+    }
+
+    @Override
+    public void close()
+    {
+        FileUtils.closeQuietly(Arrays.asList(triePrefixSearcher, cursor, rowIdToToken));
+    }
+
+    private PrimaryKey supplier(long sstableRowId)
+    {
+        try
+        {
+            cursor.seekToPointId(sstableRowId);
+            ByteSource.Peekable peekable = ByteSource.peekable(cursor.term().asComparableBytes(ByteComparable.Version.OSS42));
+
+            Token token = partitioner.getTokenFactory().fromComparableBytes(ByteSourceInverse.nextComponentSource(peekable),
+                                                                            ByteComparable.Version.OSS42);
+            byte[] keyBytes = ByteSourceInverse.getUnescapedBytes(ByteSourceInverse.nextComponentSource(peekable));
+
+            if (keyBytes == null)
+                return primaryKeyFactory.createTokenOnly(token);
+
+            DecoratedKey partitionKey = new BufferDecoratedKey(token, ByteBuffer.wrap(keyBytes));
+
+            Clustering<?> clustering = clusteringComparator.size() == 0
+                                       ? Clustering.EMPTY
+                                       : clusteringComparator.clusteringFromByteComparable(ByteBufferAccessor.instance,
+                                                                                           v -> ByteSourceInverse.nextComponentSource(peekable));
+
+            return primaryKeyFactory.create(partitionKey, clustering);
+        }
+        catch (IOException e)
+        {
+            throw Throwables.cleaned(e);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/SAICodecUtils.java b/src/java/org/apache/cassandra/index/sai/disk/v1/SAICodecUtils.java
new file mode 100644
index 0000000000..72d1a55539
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/SAICodecUtils.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.IOException;
+
+import org.apache.cassandra.index.sai.disk.format.Version;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+
+import static org.apache.lucene.codecs.CodecUtil.CODEC_MAGIC;
+import static org.apache.lucene.codecs.CodecUtil.FOOTER_MAGIC;
+
+public class SAICodecUtils
+{
+    public static final String FOOTER_POINTER = "footerPointer";
+
+    public static void writeHeader(IndexOutput out) throws IOException
+    {
+        out.writeInt(CODEC_MAGIC);
+        out.writeString(Version.LATEST.toString());
+    }
+
+    public static void writeFooter(IndexOutput out) throws IOException
+    {
+        out.writeInt(FOOTER_MAGIC);
+        out.writeInt(0);
+        writeCRC(out);
+    }
+
+    public static void checkHeader(DataInput in) throws IOException
+    {
+        final int actualMagic = in.readInt();
+        if (actualMagic != CODEC_MAGIC)
+        {
+            throw new CorruptIndexException("codec header mismatch: actual header=" + actualMagic + " vs expected header=" + CODEC_MAGIC, in);
+        }
+        final Version actualVersion = Version.parse(in.readString());
+        if (!actualVersion.onOrAfter(Version.EARLIEST))
+        {
+            throw new IOException("Unsupported version: " + actualVersion);
+        }
+    }
+
+    public static void checkFooter(ChecksumIndexInput in) throws IOException
+    {
+        validateFooter(in, false);
+        long actualChecksum = in.getChecksum();
+        long expectedChecksum = readCRC(in);
+        if (expectedChecksum != actualChecksum)
+        {
+            throw new CorruptIndexException("checksum failed (hardware problem?) : expected=" + Long.toHexString(expectedChecksum) +
+                                            " actual=" + Long.toHexString(actualChecksum), in);
+        }
+    }
+
+    public static void validate(IndexInput input) throws IOException
+    {
+        checkHeader(input);
+        validateFooterAndResetPosition(input);
+    }
+
+    public static void validate(IndexInput input, long footerPointer) throws IOException
+    {
+        checkHeader(input);
+
+        long current = input.getFilePointer();
+        input.seek(footerPointer);
+        validateFooter(input, true);
+
+        input.seek(current);
+    }
+
+    public static void validateChecksum(IndexInput input) throws IOException
+    {
+        CodecUtil.checksumEntireFile(input);
+    }
+
+    // Copied from Lucene PackedInts as they are not public
+
+    public static int checkBlockSize(int blockSize, int minBlockSize, int maxBlockSize)
+    {
+        if (blockSize >= minBlockSize && blockSize <= maxBlockSize)
+        {
+            if ((blockSize & blockSize - 1) != 0)
+            {
+                throw new IllegalArgumentException("blockSize must be a power of two, got " + blockSize);
+            }
+            else
+            {
+                return Integer.numberOfTrailingZeros(blockSize);
+            }
+        }
+        else
+        {
+            throw new IllegalArgumentException("blockSize must be >= " + minBlockSize + " and <= " + maxBlockSize + ", got " + blockSize);
+        }
+    }
+
+    public static int numBlocks(long size, int blockSize)
+    {
+        if (size < 0)
+            throw new IllegalArgumentException("size cannot be negative");
+
+        int numBlocks = (int)(size / (long)blockSize) + (size % (long)blockSize == 0L ? 0 : 1);
+        if ((long)numBlocks * (long)blockSize < size)
+        {
+            throw new IllegalArgumentException("size is too large for this block size");
+        }
+        else
+        {
+            return numBlocks;
+        }
+    }
+
+    // Copied from Lucene BlockPackedReaderIterator as they are not public
+
+    /**
+     * Same as DataInput.readVLong but supports negative values
+     */
+    public static long readVLong(DataInput in) throws IOException
+    {
+        byte b = in.readByte();
+        if (b >= 0) return b;
+        long i = b & 0x7FL;
+        b = in.readByte();
+        i |= (b & 0x7FL) << 7;
+        if (b >= 0) return i;
+        b = in.readByte();
+        i |= (b & 0x7FL) << 14;
+        if (b >= 0) return i;
+        b = in.readByte();
+        i |= (b & 0x7FL) << 21;
+        if (b >= 0) return i;
+        b = in.readByte();
+        i |= (b & 0x7FL) << 28;
+        if (b >= 0) return i;
+        b = in.readByte();
+        i |= (b & 0x7FL) << 35;
+        if (b >= 0) return i;
+        b = in.readByte();
+        i |= (b & 0x7FL) << 42;
+        if (b >= 0) return i;
+        b = in.readByte();
+        i |= (b & 0x7FL) << 49;
+        if (b >= 0) return i;
+        b = in.readByte();
+        i |= (b & 0xFFL) << 56;
+        return i;
+    }
+
+    public static void validateFooterAndResetPosition(IndexInput in) throws IOException
+    {
+        long position = in.getFilePointer();
+        long fileLength = in.length();
+        long footerLength = CodecUtil.footerLength();
+        long footerPosition = fileLength - footerLength;
+
+        if (footerPosition < 0)
+        {
+            throw new CorruptIndexException("invalid codec footer (file truncated?): file length=" + fileLength + ", footer length=" + footerLength, in);
+        }
+
+        in.seek(footerPosition);
+        validateFooter(in, false);
+        in.seek(position);
+    }
+
+    /**
+     * Copied from org.apache.lucene.codecs.CodecUtil.validateFooter(IndexInput).
+     *
+     * If the file is segmented then the footer can exist in the middle of the file
+     * so, we shouldn't check that the footer size is correct, we just check that the
+     * footer values are correct.
+     */
+    private static void validateFooter(IndexInput in, boolean segmented) throws IOException
+    {
+        long remaining = in.length() - in.getFilePointer();
+        long expected = CodecUtil.footerLength();
+
+        if (!segmented)
+        {
+            if (remaining < expected)
+            {
+                throw new CorruptIndexException("misplaced codec footer (file truncated?): remaining=" + remaining + ", expected=" + expected + ", fp=" + in.getFilePointer(), in);
+            }
+            else if (remaining > expected)
+            {
+                throw new CorruptIndexException("misplaced codec footer (file extended?): remaining=" + remaining + ", expected=" + expected + ", fp=" + in.getFilePointer(), in);
+            }
+        }
+
+        final int magic = in.readInt();
+
+        if (magic != FOOTER_MAGIC)
+        {
+            throw new CorruptIndexException("codec footer mismatch (file truncated?): actual footer=" + magic + " vs expected footer=" + FOOTER_MAGIC, in);
+        }
+
+        final int algorithmID = in.readInt();
+
+        if (algorithmID != 0)
+        {
+            throw new CorruptIndexException("codec footer mismatch: unknown algorithmID: " + algorithmID, in);
+        }
+    }
+
+    // Copied from Lucene CodecUtil as they are not public
+
+    /**
+     * Writes CRC32 value as a 64-bit long to the output.
+     * @throws IllegalStateException if CRC is formatted incorrectly (wrong bits set)
+     * @throws IOException if an i/o error occurs
+     */
+    private static void writeCRC(IndexOutput output) throws IOException
+    {
+        long value = output.getChecksum();
+        if ((value & 0xFFFFFFFF00000000L) != 0)
+        {
+            throw new IllegalStateException("Illegal CRC-32 checksum: " + value + " (resource=" + output + ')');
+        }
+        output.writeLong(value);
+    }
+
+    /**
+     * Reads CRC32 value as a 64-bit long from the input.
+     * @throws CorruptIndexException if CRC is formatted incorrectly (wrong bits set)
+     * @throws IOException if an i/o error occurs
+     */
+    private static long readCRC(IndexInput input) throws IOException
+    {
+        long value = input.readLong();
+        if ((value & 0xFFFFFFFF00000000L) != 0)
+        {
+            throw new CorruptIndexException("Illegal CRC-32 checksum: " + value, input);
+        }
+        return value;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableComponentsWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableComponentsWriter.java
new file mode 100644
index 0000000000..8845f27300
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableComponentsWriter.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.index.sai.disk.PerSSTableIndexWriter;
+import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.io.IndexOutputWriter;
+import org.apache.cassandra.index.sai.disk.v1.bitpack.NumericValuesWriter;
+import org.apache.cassandra.index.sai.disk.v1.sortedterms.SortedTermsWriter;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.lucene.util.IOUtils;
+
+public class SSTableComponentsWriter implements PerSSTableIndexWriter
+{
+    protected static final Logger logger = LoggerFactory.getLogger(SSTableComponentsWriter.class);
+
+    private final IndexDescriptor indexDescriptor;
+    private final MetadataWriter metadataWriter;
+    private final NumericValuesWriter tokenWriter;
+    private final SortedTermsWriter sortedTermsWriter;
+
+    @SuppressWarnings({"resource", "RedundantSuppression"})
+    public SSTableComponentsWriter(IndexDescriptor indexDescriptor) throws IOException
+    {
+        this.indexDescriptor = indexDescriptor;
+        this.metadataWriter = new MetadataWriter(indexDescriptor.openPerSSTableOutput(IndexComponent.GROUP_META));
+        this.tokenWriter = new NumericValuesWriter(indexDescriptor.componentName(IndexComponent.TOKEN_VALUES),
+                                                   indexDescriptor.openPerSSTableOutput(IndexComponent.TOKEN_VALUES),
+                                                   metadataWriter, false);
+        IndexOutputWriter primaryKeyTrieWriter = indexDescriptor.openPerSSTableOutput(IndexComponent.PRIMARY_KEY_TRIE);
+        IndexOutputWriter primaryKeyBlocksWriter = indexDescriptor.openPerSSTableOutput(IndexComponent.PRIMARY_KEY_BLOCKS);
+        NumericValuesWriter primaryKeyBlockOffsetWriter = new NumericValuesWriter(indexDescriptor.componentName(IndexComponent.PRIMARY_KEY_BLOCK_OFFSETS),
+                                                     indexDescriptor.openPerSSTableOutput(IndexComponent.PRIMARY_KEY_BLOCK_OFFSETS),
+                                                     metadataWriter, true);
+        this.sortedTermsWriter = new SortedTermsWriter(indexDescriptor.componentName(IndexComponent.PRIMARY_KEY_BLOCKS),
+                                                       metadataWriter,
+                                                       primaryKeyBlocksWriter,
+                                                       primaryKeyBlockOffsetWriter,
+                                                       primaryKeyTrieWriter);
+    }
+
+    @Override
+    public void nextRow(PrimaryKey primaryKey) throws IOException
+    {
+        tokenWriter.add(primaryKey.token().getLongValue());
+        sortedTermsWriter.add(primaryKey::asComparableBytes);
+    }
+
+    @Override
+    public void complete() throws IOException
+    {
+        try
+        {
+            indexDescriptor.createComponentOnDisk(IndexComponent.GROUP_COMPLETION_MARKER);
+        }
+        finally
+        {
+            IOUtils.close(tokenWriter, sortedTermsWriter, metadataWriter);
+        }
+    }
+
+    @Override
+    public void abort()
+    {
+        logger.debug(indexDescriptor.logMessage("Aborting per-SSTable index component writer for {}..."), indexDescriptor.sstableDescriptor);
+        indexDescriptor.deletePerSSTableIndexComponents();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java
new file mode 100644
index 0000000000..fc12c41c85
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BooleanSupplier;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import com.google.common.base.Stopwatch;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.analyzer.AbstractAnalyzer;
+import org.apache.cassandra.index.sai.disk.PerColumnIndexWriter;
+import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder;
+import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
+import org.apache.cassandra.index.sai.utils.NamedMemoryLimiter;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.NoSpamLogger;
+
+/**
+ * Column index writer that accumulates (on-heap) indexed data from a compacted SSTable as it's being flushed to disk.
+ */
+@NotThreadSafe
+public class SSTableIndexWriter implements PerColumnIndexWriter
+{
+    private static final Logger logger = LoggerFactory.getLogger(SSTableIndexWriter.class);
+    private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES);
+
+    public static final int MAX_STRING_TERM_SIZE = Integer.getInteger("cassandra.sai.max_string_term_size_kb", 1) * 1024;
+    public static final int MAX_FROZEN_TERM_SIZE = Integer.getInteger("cassandra.sai.max_frozen_term_size_kb", 5) * 1024;
+    public static final String TERM_OVERSIZE_MESSAGE = "Can't add term of column {} to index for key: {}, term size {} " +
+                                                       "max allowed size {}, use analyzed = true (if not yet set) for that column.";
+
+    private final IndexDescriptor indexDescriptor;
+    private final IndexContext indexContext;
+    private final int nowInSec = FBUtilities.nowInSeconds();
+    private final AbstractAnalyzer analyzer;
+    private final NamedMemoryLimiter limiter;
+    private final int maxTermSize;
+    private final BooleanSupplier isIndexValid;
+    private final List<SegmentMetadata> segments = new ArrayList<>();
+
+    private boolean aborted = false;
+    private SegmentBuilder currentBuilder;
+
+    public SSTableIndexWriter(IndexDescriptor indexDescriptor, IndexContext indexContext, NamedMemoryLimiter limiter, BooleanSupplier isIndexValid)
+    {
+        this.indexDescriptor = indexDescriptor;
+        this.indexContext = indexContext;
+        this.analyzer = indexContext.getIndexAnalyzerFactory().create();
+        this.limiter = limiter;
+        this.isIndexValid = isIndexValid;
+        this.maxTermSize = indexContext.isFrozen() ? MAX_FROZEN_TERM_SIZE : MAX_STRING_TERM_SIZE;
+    }
+
+    @Override
+    public void addRow(PrimaryKey key, Row row, long sstableRowId) throws IOException
+    {
+        if (maybeAbort())
+            return;
+
+        if (indexContext.isNonFrozenCollection())
+        {
+            Iterator<ByteBuffer> valueIterator = indexContext.getValuesOf(row, nowInSec);
+            if (valueIterator != null)
+            {
+                while (valueIterator.hasNext())
+                {
+                    ByteBuffer value = valueIterator.next();
+                    addTerm(TypeUtil.asIndexBytes(value.duplicate(), indexContext.getValidator()), key, sstableRowId, indexContext.getValidator());
+                }
+            }
+        }
+        else
+        {
+            ByteBuffer value = indexContext.getValueOf(key.partitionKey(), row, nowInSec);
+            if (value != null)
+                addTerm(TypeUtil.asIndexBytes(value.duplicate(), indexContext.getValidator()), key, sstableRowId, indexContext.getValidator());
+        }
+    }
+
+    @Override
+    public void complete(Stopwatch stopwatch) throws IOException
+    {
+        if (maybeAbort())
+            return;
+
+        long start = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+        long elapsed;
+
+        boolean emptySegment = currentBuilder == null || currentBuilder.isEmpty();
+        logger.debug(indexContext.logMessage("Completing index flush with {}buffered data..."), emptySegment ? "no " : "");
+
+        try
+        {
+            // parts are present but there is something still in memory, let's flush that inline
+            if (!emptySegment)
+            {
+                flushSegment();
+                elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS);
+                logger.debug(indexContext.logMessage("Completed flush of final segment for SSTable {}. Duration: {} ms. Total elapsed: {} ms"),
+                             indexDescriptor.sstableDescriptor,
+                             elapsed - start,
+                             elapsed);
+            }
+
+            // Even an empty segment may carry some fixed memory, so remove it:
+            if (currentBuilder != null)
+            {
+                long bytesAllocated = currentBuilder.totalBytesAllocated();
+                long globalBytesUsed = currentBuilder.release(indexContext);
+                logger.debug(indexContext.logMessage("Flushing final segment for SSTable {} released {}. Global segment memory usage now at {}."),
+                             indexDescriptor.sstableDescriptor, FBUtilities.prettyPrintMemory(bytesAllocated), FBUtilities.prettyPrintMemory(globalBytesUsed));
+            }
+
+            writeSegmentsMetadata();
+            indexDescriptor.createComponentOnDisk(IndexComponent.COLUMN_COMPLETION_MARKER, indexContext);
+        }
+        finally
+        {
+            if (indexContext.getIndexMetrics() != null)
+            {
+                indexContext.getIndexMetrics().segmentsPerCompaction.update(segments.size());
+                segments.clear();
+                indexContext.getIndexMetrics().compactionCount.inc();
+            }
+        }
+    }
+
+    @Override
+    public void abort(Throwable cause)
+    {
+        aborted = true;
+
+        logger.warn(indexContext.logMessage("Aborting SSTable index flush for {}..."), indexDescriptor.sstableDescriptor, cause);
+
+        // It's possible for the current builder to be unassigned after we flush a final segment.
+        if (currentBuilder != null)
+        {
+            // If an exception is thrown out of any writer operation prior to successful segment
+            // flush, we will end up here, and we need to free up builder memory tracked by the limiter:
+            long allocated = currentBuilder.totalBytesAllocated();
+            long globalBytesUsed = currentBuilder.release(indexContext);
+            logger.debug(indexContext.logMessage("Aborting index writer for SSTable {} released {}. Global segment memory usage now at {}."),
+                         indexDescriptor.sstableDescriptor, FBUtilities.prettyPrintMemory(allocated), FBUtilities.prettyPrintMemory(globalBytesUsed));
+        }
+
+        indexDescriptor.deleteColumnIndex(indexContext);
+    }
+
+    /**
+     * abort current write if index is dropped
+     *
+     * @return true if current write is aborted.
+     */
+    private boolean maybeAbort()
+    {
+        if (aborted)
+            return true;
+
+        if (isIndexValid.getAsBoolean())
+            return false;
+
+        abort(new RuntimeException(String.format("index %s is dropped", indexContext.getIndexName())));
+        return true;
+    }
+
+    private void addTerm(ByteBuffer term, PrimaryKey key, long sstableRowId, AbstractType<?> type) throws IOException
+    {
+        if (term.remaining() >= maxTermSize)
+        {
+            noSpamLogger.warn(indexContext.logMessage(TERM_OVERSIZE_MESSAGE),
+                              indexContext.getColumnName(),
+                              indexContext.keyValidator().getString(key.partitionKey().getKey()),
+                              FBUtilities.prettyPrintMemory(term.remaining()),
+                              FBUtilities.prettyPrintMemory(maxTermSize));
+            return;
+        }
+
+        if (currentBuilder == null)
+        {
+            currentBuilder = newSegmentBuilder();
+        }
+        else if (shouldFlush(sstableRowId))
+        {
+            flushSegment();
+            currentBuilder = newSegmentBuilder();
+        }
+
+        if (term.remaining() == 0) return;
+
+        if (!TypeUtil.isLiteral(type))
+        {
+            limiter.increment(currentBuilder.add(term, key, sstableRowId));
+        }
+        else
+        {
+            analyzer.reset(term);
+            try
+            {
+                while (analyzer.hasNext())
+                {
+                    ByteBuffer tokenTerm = analyzer.next();
+                    limiter.increment(currentBuilder.add(tokenTerm, key, sstableRowId));
+                }
+            }
+            finally
+            {
+                analyzer.end();
+            }
+        }
+    }
+
+    private boolean shouldFlush(long sstableRowId)
+    {
+        // If we've hit the minimum flush size and, we've breached the global limit, flush a new segment:
+        boolean reachMemoryLimit = limiter.usageExceedsLimit() && currentBuilder.hasReachedMinimumFlushSize();
+
+        if (reachMemoryLimit)
+        {
+            logger.debug(indexContext.logMessage("Global limit of {} and minimum flush size of {} exceeded. " +
+                                            "Current builder usage is {} for {} cells. Global Usage is {}. Flushing..."),
+                         FBUtilities.prettyPrintMemory(limiter.limitBytes()),
+                         FBUtilities.prettyPrintMemory(currentBuilder.getMinimumFlushBytes()),
+                         FBUtilities.prettyPrintMemory(currentBuilder.totalBytesAllocated()),
+                         currentBuilder.getRowCount(),
+                         FBUtilities.prettyPrintMemory(limiter.currentBytesUsed()));
+        }
+
+        return reachMemoryLimit || currentBuilder.exceedsSegmentLimit(sstableRowId);
+    }
+
+    private void flushSegment() throws IOException
+    {
+        long start = Clock.Global.nanoTime();
+
+        try
+        {
+            long bytesAllocated = currentBuilder.totalBytesAllocated();
+
+            SegmentMetadata segmentMetadata = currentBuilder.flush(indexDescriptor, indexContext);
+
+            long flushMillis = Math.max(1, TimeUnit.NANOSECONDS.toMillis(Clock.Global.nanoTime() - start));
+
+            if (segmentMetadata != null)
+            {
+                segments.add(segmentMetadata);
+
+                double rowCount = segmentMetadata.numRows;
+                if (indexContext.getIndexMetrics() != null)
+                    indexContext.getIndexMetrics().compactionSegmentCellsPerSecond.update((long)(rowCount / flushMillis * 1000.0));
+
+                double segmentBytes = segmentMetadata.componentMetadatas.indexSize();
+                if (indexContext.getIndexMetrics() != null)
+                    indexContext.getIndexMetrics().compactionSegmentBytesPerSecond.update((long)(segmentBytes / flushMillis * 1000.0));
+
+                logger.debug(indexContext.logMessage("Flushed segment with {} cells for a total of {} in {} ms."),
+                             (long) rowCount, FBUtilities.prettyPrintMemory((long) segmentBytes), flushMillis);
+            }
+
+            // Builder memory is released against the limiter at the conclusion of a successful
+            // flush. Note that any failure that occurs before this (even in term addition) will
+            // actuate this column writer's abort logic from the parent SSTable-level writer, and
+            // that abort logic will release the current builder's memory against the limiter.
+            long globalBytesUsed = currentBuilder.release(indexContext);
+            currentBuilder = null;
+            logger.debug(indexContext.logMessage("Flushing index segment for SSTable {} released {}. Global segment memory usage now at {}."),
+                         indexDescriptor.sstableDescriptor, FBUtilities.prettyPrintMemory(bytesAllocated), FBUtilities.prettyPrintMemory(globalBytesUsed));
+
+        }
+        catch (Throwable t)
+        {
+            logger.error(indexContext.logMessage("Failed to build index for SSTable {}."), indexDescriptor.sstableDescriptor, t);
+            indexDescriptor.deleteColumnIndex(indexContext);
+            indexContext.getIndexMetrics().segmentFlushErrors.inc();
+            throw t;
+        }
+    }
+
+    private void writeSegmentsMetadata() throws IOException
+    {
+        if (segments.isEmpty())
+            return;
+
+        try (MetadataWriter writer = new MetadataWriter(indexDescriptor.openPerIndexOutput(IndexComponent.META, indexContext)))
+        {
+            SegmentMetadata.write(writer, segments);
+        }
+        catch (IOException e)
+        {
+            abort(e);
+            throw e;
+        }
+    }
+
+    private SegmentBuilder newSegmentBuilder()
+    {
+        SegmentBuilder builder = new SegmentBuilder.RAMStringSegmentBuilder(indexContext.getValidator(), limiter);
+
+        long globalBytesUsed = limiter.increment(builder.totalBytesAllocated());
+        logger.debug(indexContext.logMessage("Created new segment builder while flushing SSTable {}. Global segment memory usage now at {}."),
+                     indexDescriptor.sstableDescriptor,
+                     FBUtilities.prettyPrintMemory(globalBytesUsed));
+
+        return builder;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/V1OnDiskFormat.java b/src/java/org/apache/cassandra/index/sai/disk/v1/V1OnDiskFormat.java
new file mode 100644
index 0000000000..c55d4b07ef
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/V1OnDiskFormat.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.Gauge;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.StorageAttachedIndex;
+import org.apache.cassandra.index.sai.disk.PerColumnIndexWriter;
+import org.apache.cassandra.index.sai.disk.PerSSTableIndexWriter;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.disk.RowMapping;
+import org.apache.cassandra.index.sai.disk.SSTableIndex;
+import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.format.OnDiskFormat;
+import org.apache.cassandra.index.sai.disk.v1.segment.SegmentBuilder;
+import org.apache.cassandra.index.sai.metrics.AbstractMetrics;
+import org.apache.cassandra.index.sai.utils.NamedMemoryLimiter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.metrics.CassandraMetricsRegistry;
+import org.apache.cassandra.metrics.DefaultNameFactory;
+import org.apache.lucene.store.IndexInput;
+
+import static org.apache.cassandra.utils.FBUtilities.prettyPrintMemory;
+
+public class V1OnDiskFormat implements OnDiskFormat
+{
+    private static final Logger logger = LoggerFactory.getLogger(V1OnDiskFormat.class);
+
+    private static final Set<IndexComponent> PER_SSTABLE_COMPONENTS = EnumSet.of(IndexComponent.GROUP_COMPLETION_MARKER,
+                                                                                 IndexComponent.GROUP_META,
+                                                                                 IndexComponent.TOKEN_VALUES,
+                                                                                 IndexComponent.PRIMARY_KEY_TRIE,
+                                                                                 IndexComponent.PRIMARY_KEY_BLOCKS,
+                                                                                 IndexComponent.PRIMARY_KEY_BLOCK_OFFSETS);
+    private static final Set<IndexComponent> LITERAL_COMPONENTS = EnumSet.of(IndexComponent.COLUMN_COMPLETION_MARKER,
+                                                                             IndexComponent.META,
+                                                                             IndexComponent.TERMS_DATA,
+                                                                             IndexComponent.POSTING_LISTS);
+
+    /**
+     * Global limit on heap consumed by all index segment building that occurs outside the context of Memtable flush.
+     *
+     * Note that to avoid flushing small index segments, a segment is only flushed when
+     * both the global size of all building segments has breached the limit and the size of the
+     * segment in question reaches (segment_write_buffer_space_mb / # currently building column indexes).
+     *
+     * ex. If there is only one column index building, it can buffer up to segment_write_buffer_space_mb.
+     *
+     * ex. If there is one column index building per table across 8 compactors, each index will be
+     *     eligible to flush once it reaches (segment_write_buffer_space_mb / 8) MBs.
+     */
+    public static final long SEGMENT_BUILD_MEMORY_LIMIT = DatabaseDescriptor.getSAISegmentWriteBufferSpace().toBytes();
+
+    public static final NamedMemoryLimiter SEGMENT_BUILD_MEMORY_LIMITER = new NamedMemoryLimiter(SEGMENT_BUILD_MEMORY_LIMIT,
+                                                                                                 "Storage Attached Index Segment Builder");
+
+    static
+    {
+        CassandraMetricsRegistry.MetricName bufferSpaceUsed = DefaultNameFactory.createMetricName(AbstractMetrics.TYPE, "SegmentBufferSpaceUsedBytes", null);
+        CassandraMetricsRegistry.Metrics.register(bufferSpaceUsed, (Gauge<Long>) SEGMENT_BUILD_MEMORY_LIMITER::currentBytesUsed);
+
+        CassandraMetricsRegistry.MetricName bufferSpaceLimit = DefaultNameFactory.createMetricName(AbstractMetrics.TYPE, "SegmentBufferSpaceLimitBytes", null);
+        CassandraMetricsRegistry.Metrics.register(bufferSpaceLimit, (Gauge<Long>) () -> SEGMENT_BUILD_MEMORY_LIMIT);
+
+        CassandraMetricsRegistry.MetricName buildsInProgress = DefaultNameFactory.createMetricName(AbstractMetrics.TYPE, "ColumnIndexBuildsInProgress", null);
+        CassandraMetricsRegistry.Metrics.register(buildsInProgress, (Gauge<Integer>) SegmentBuilder::getActiveBuilderCount);
+    }
+
+    public static final V1OnDiskFormat instance = new V1OnDiskFormat();
+
+    protected V1OnDiskFormat()
+    {}
+
+    @Override
+    public PrimaryKeyMap.Factory newPrimaryKeyMapFactory(IndexDescriptor indexDescriptor, SSTableReader sstable)
+    {
+        return new RowAwarePrimaryKeyMap.RowAwarePrimaryKeyMapFactory(indexDescriptor, sstable);
+    }
+
+    @Override
+    public SSTableIndex newSSTableIndex(SSTableContext sstableContext, IndexContext indexContext)
+    {
+        return new V1SSTableIndex(sstableContext, indexContext);
+    }
+
+    @Override
+    public PerSSTableIndexWriter newPerSSTableIndexWriter(IndexDescriptor indexDescriptor) throws IOException
+    {
+        return new SSTableComponentsWriter(indexDescriptor);
+    }
+
+    @Override
+    public PerColumnIndexWriter newPerColumnIndexWriter(StorageAttachedIndex index,
+                                                        IndexDescriptor indexDescriptor,
+                                                        LifecycleNewTracker tracker,
+                                                        RowMapping rowMapping)
+    {
+        // If we're not flushing, or we haven't yet started the initialization build, flush from SSTable contents.
+        if (tracker.opType() != OperationType.FLUSH || !index.isInitBuildStarted())
+        {
+            NamedMemoryLimiter limiter = SEGMENT_BUILD_MEMORY_LIMITER;
+            logger.info(index.getIndexContext().logMessage("Starting a compaction index build. Global segment memory usage: {}"),
+                        prettyPrintMemory(limiter.currentBytesUsed()));
+
+            return new SSTableIndexWriter(indexDescriptor, index.getIndexContext(), limiter, index.isIndexValid());
+        }
+
+        return new MemtableIndexWriter(index.getIndexContext().getMemtableIndexManager().getPendingMemtableIndex(tracker),
+                                       indexDescriptor,
+                                       index.getIndexContext(),
+                                       rowMapping);
+    }
+
+    @Override
+    public boolean isPerSSTableIndexBuildComplete(IndexDescriptor indexDescriptor)
+    {
+        return indexDescriptor.hasComponent(IndexComponent.GROUP_COMPLETION_MARKER);
+    }
+
+    @Override
+    public boolean isPerColumnIndexBuildComplete(IndexDescriptor indexDescriptor, IndexContext indexContext)
+    {
+        return indexDescriptor.hasComponent(IndexComponent.GROUP_COMPLETION_MARKER) &&
+               indexDescriptor.hasComponent(IndexComponent.COLUMN_COMPLETION_MARKER, indexContext);
+    }
+
+    @Override
+    public boolean validatePerSSTableIndexComponents(IndexDescriptor indexDescriptor, boolean checksum)
+    {
+        for (IndexComponent indexComponent : perSSTableIndexComponents())
+        {
+            if (isNotBuildCompletionMarker(indexComponent))
+            {
+                try (IndexInput input = indexDescriptor.openPerSSTableInput(indexComponent))
+                {
+                    if (checksum)
+                        SAICodecUtils.validateChecksum(input);
+                    else
+                        SAICodecUtils.validate(input);
+                }
+                catch (Throwable e)
+                {
+                    if (logger.isDebugEnabled())
+                    {
+                        logger.debug(indexDescriptor.logMessage("{} failed for index component {} on SSTable {}"),
+                                     (checksum ? "Checksum validation" : "Validation"),
+                                     indexComponent,
+                                     indexDescriptor.sstableDescriptor);
+                    }
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public boolean validatePerColumnIndexComponents(IndexDescriptor indexDescriptor, IndexContext indexContext, boolean checksum)
+    {
+        for (IndexComponent indexComponent : perColumnIndexComponents(indexContext))
+        {
+            if (isNotBuildCompletionMarker(indexComponent))
+            {
+                try (IndexInput input = indexDescriptor.openPerIndexInput(indexComponent, indexContext))
+                {
+                    if (checksum)
+                        SAICodecUtils.validateChecksum(input);
+                    else
+                        SAICodecUtils.validate(input);
+                }
+                catch (Throwable e)
+                {
+                    if (logger.isDebugEnabled())
+                    {
+                        logger.debug(indexDescriptor.logMessage("{} failed for index component {} on SSTable {}"),
+                                     (checksum ? "Checksum validation" : "Validation"),
+                                     indexComponent,
+                                     indexDescriptor.sstableDescriptor);
+                    }
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public Set<IndexComponent> perSSTableIndexComponents()
+    {
+        return PER_SSTABLE_COMPONENTS;
+    }
+
+    @Override
+    public Set<IndexComponent> perColumnIndexComponents(IndexContext indexContext)
+    {
+        return LITERAL_COMPONENTS;
+    }
+
+    @Override
+    public int openFilesPerSSTableIndex()
+    {
+        // For the V1 format there are always 4 open files per SSTable - token values, primary key trie,
+        // primary key blocks, primary key block offsets
+        return 4;
+    }
+
+    @Override
+    public int openFilesPerColumnIndex(IndexContext indexContext)
+    {
+        // For the V1 format there are always 2 open files per index - index (kdtree or terms) + postings
+        return 2;
+    }
+
+    protected boolean isNotBuildCompletionMarker(IndexComponent indexComponent)
+    {
+        return indexComponent != IndexComponent.GROUP_COMPLETION_MARKER &&
+               indexComponent != IndexComponent.COLUMN_COMPLETION_MARKER;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/V1SSTableIndex.java b/src/java/org/apache/cassandra/index/sai/disk/v1/V1SSTableIndex.java
new file mode 100644
index 0000000000..fd89b302c8
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/V1SSTableIndex.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.db.virtual.SimpleDataSet;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.disk.SSTableIndex;
+import org.apache.cassandra.index.sai.disk.v1.segment.Segment;
+import org.apache.cassandra.index.sai.disk.v1.segment.SegmentMetadata;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.Throwables;
+
+import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.CELL_COUNT;
+import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.COLUMN_NAME;
+import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.COMPONENT_METADATA;
+import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.END_TOKEN;
+import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.MAX_SSTABLE_ROW_ID;
+import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.MAX_TERM;
+import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.MIN_SSTABLE_ROW_ID;
+import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.MIN_TERM;
+import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.START_TOKEN;
+import static org.apache.cassandra.index.sai.virtual.SegmentsSystemView.TABLE_NAME;
+
+/**
+ * A version specific implementation of the {@link SSTableIndex} where the
+ * index is segmented
+ */
+public class V1SSTableIndex extends SSTableIndex
+{
+    private final ImmutableList<Segment> segments;
+    private final List<SegmentMetadata> metadatas;
+    private final AbstractBounds<PartitionPosition> bounds;
+    private final ByteBuffer minTerm;
+    private final ByteBuffer maxTerm;
+    private final long minSSTableRowId, maxSSTableRowId;
+    private final long numRows;
+
+    private PerColumnIndexFiles indexFiles;
+
+    public V1SSTableIndex(SSTableContext sstableContext, IndexContext indexContext)
+    {
+        super(sstableContext, indexContext);
+
+        try
+        {
+            this.indexFiles = new PerColumnIndexFiles(sstableContext.indexDescriptor, indexContext);
+
+            ImmutableList.Builder<Segment> segmentsBuilder = ImmutableList.builder();
+
+            final MetadataSource source = MetadataSource.loadColumnMetadata(sstableContext.indexDescriptor, indexContext);
+
+            metadatas = SegmentMetadata.load(source, sstableContext.indexDescriptor.primaryKeyFactory);
+
+            for (SegmentMetadata metadata : metadatas)
+            {
+                segmentsBuilder.add(new Segment(indexContext, sstableContext, indexFiles, metadata));
+            }
+
+            segments = segmentsBuilder.build();
+            assert !segments.isEmpty();
+
+            DecoratedKey minKey = metadatas.get(0).minKey.partitionKey();
+            DecoratedKey maxKey = metadatas.get(metadatas.size() - 1).maxKey.partitionKey();
+
+            this.bounds = AbstractBounds.bounds(minKey, true, maxKey, true);
+
+            this.minTerm = metadatas.stream().map(m -> m.minTerm).min(TypeUtil.comparator(indexContext.getValidator())).orElse(null);
+            this.maxTerm = metadatas.stream().map(m -> m.maxTerm).max(TypeUtil.comparator(indexContext.getValidator())).orElse(null);
+
+            this.numRows = metadatas.stream().mapToLong(m -> m.numRows).sum();
+
+            this.minSSTableRowId = metadatas.get(0).minSSTableRowId;
+            this.maxSSTableRowId = metadatas.get(metadatas.size() - 1).maxSSTableRowId;
+        }
+        catch (Throwable t)
+        {
+            FileUtils.closeQuietly(indexFiles);
+            FileUtils.closeQuietly(sstableContext);
+            throw Throwables.unchecked(t);
+        }
+    }
+
+    @Override
+    public long indexFileCacheSize()
+    {
+        return segments.stream().mapToLong(Segment::indexFileCacheSize).sum();
+    }
+
+    @Override
+    public long getRowCount()
+    {
+        return numRows;
+    }
+
+    @Override
+    public long minSSTableRowId()
+    {
+        return minSSTableRowId;
+    }
+
+    @Override
+    public long maxSSTableRowId()
+    {
+        return maxSSTableRowId;
+    }
+
+    @Override
+    public ByteBuffer minTerm()
+    {
+        return minTerm;
+    }
+
+    @Override
+    public ByteBuffer maxTerm()
+    {
+        return maxTerm;
+    }
+
+    @Override
+    public AbstractBounds<PartitionPosition> bounds()
+    {
+        return bounds;
+    }
+
+    @Override
+    public List<KeyRangeIterator> search(Expression expression,
+                                         AbstractBounds<PartitionPosition> keyRange,
+                                         QueryContext context) throws IOException
+    {
+        List<KeyRangeIterator> segmentIterators = new ArrayList<>();
+
+        for (Segment segment : segments)
+        {
+            if (segment.intersects(keyRange))
+            {
+                segmentIterators.add(segment.search(expression, context));
+            }
+        }
+
+        return segmentIterators;
+    }
+
+    @Override
+    public void populateSegmentView(SimpleDataSet dataset)
+    {
+        SSTableReader sstable = getSSTable();
+        Token.TokenFactory tokenFactory = sstable.metadata().partitioner.getTokenFactory();
+
+        for (SegmentMetadata metadata : metadatas)
+        {
+            dataset.row(sstable.metadata().keyspace, indexContext.getIndexName(), sstable.getFilename(), metadata.rowIdOffset)
+                   .column(TABLE_NAME, sstable.descriptor.cfname)
+                   .column(COLUMN_NAME, indexContext.getColumnName())
+                   .column(CELL_COUNT, metadata.numRows)
+                   .column(MIN_SSTABLE_ROW_ID, metadata.minSSTableRowId)
+                   .column(MAX_SSTABLE_ROW_ID, metadata.maxSSTableRowId)
+                   .column(START_TOKEN, tokenFactory.toString(metadata.minKey.token()))
+                   .column(END_TOKEN, tokenFactory.toString(metadata.maxKey.token()))
+                   .column(MIN_TERM, indexContext.getValidator().getSerializer().deserialize(metadata.minTerm).toString())
+                   .column(MAX_TERM, indexContext.getValidator().getSerializer().deserialize(metadata.maxTerm).toString())
+                   .column(COMPONENT_METADATA, metadata.componentMetadatas.asMap());
+        }
+    }
+
+    @Override
+    protected void internalRelease()
+    {
+        FileUtils.closeQuietly(indexFiles);
+        FileUtils.closeQuietly(segments);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/AbstractBlockPackedReader.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/AbstractBlockPackedReader.java
new file mode 100644
index 0000000000..4d0bb40118
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/AbstractBlockPackedReader.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.bitpack;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.index.sai.disk.io.SeekingRandomAccessInput;
+import org.apache.cassandra.index.sai.disk.v1.DirectReaders;
+import org.apache.cassandra.index.sai.disk.v1.LongArray;
+import org.apache.lucene.store.IndexInput;
+
+@NotThreadSafe
+public abstract class AbstractBlockPackedReader implements LongArray
+{
+    private final int blockShift;
+    private final int blockMask;
+    private final long valueCount;
+    private final byte[] blockBitsPerValue;
+    private final SeekingRandomAccessInput input;
+
+    AbstractBlockPackedReader(IndexInput indexInput, byte[] blockBitsPerValue, int blockShift, int blockMask, long valueCount)
+    {
+        this.blockShift = blockShift;
+        this.blockMask = blockMask;
+        this.valueCount = valueCount;
+        this.input = new SeekingRandomAccessInput(indexInput);
+        this.blockBitsPerValue = blockBitsPerValue;
+    }
+
+    protected abstract long blockOffsetAt(int block);
+
+    @Override
+    public long get(final long valueIndex)
+    {
+        if (valueIndex < 0 || valueIndex >= valueCount)
+        {
+            throw new IndexOutOfBoundsException(String.format("Index should be between [0, %d), but was %d.", valueCount, valueIndex));
+        }
+
+        int blockIndex = (int) (valueIndex >>> blockShift);
+        int inBlockIndex = (int) (valueIndex & blockMask);
+        DirectReaders.Reader subReader = DirectReaders.getReaderForBitsPerValue(blockBitsPerValue[blockIndex]);
+        return delta(blockIndex, inBlockIndex) + subReader.get(input, blockOffsetAt(blockIndex), inBlockIndex);
+    }
+
+    @Override
+    public long length()
+    {
+        return valueCount;
+    }
+
+    abstract long delta(int block, int idx);
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/AbstractBlockPackedWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/AbstractBlockPackedWriter.java
new file mode 100644
index 0000000000..ead4f5986f
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/AbstractBlockPackedWriter.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.bitpack;
+
+import java.io.IOException;
+
+import org.apache.cassandra.index.sai.disk.io.RAMIndexOutput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.packed.DirectWriter;
+
+import static org.apache.cassandra.index.sai.disk.v1.SAICodecUtils.checkBlockSize;
+
+/**
+ * Modified copy of {@code org.apache.lucene.util.packed.AbstractBlockPackedWriter} to use {@link DirectWriter} for
+ * optimised reads that doesn't require seeking through the whole file to open a thread-exclusive reader.
+ */
+public abstract class AbstractBlockPackedWriter
+{
+    static final int MIN_BLOCK_SIZE = 64;
+    static final int MAX_BLOCK_SIZE = 1 << (30 - 3);
+
+    protected final IndexOutput indexOutput;
+    protected final long[] blockValues;
+    // This collects metadata specific to the block packed writer being used during the
+    // writing of the block packed data. This cached metadata is then written to the end
+    // of the data file when the block packed writer is finished.
+    protected final RAMIndexOutput blockMetaWriter;
+
+    protected int blockIndex;
+    protected boolean finished;
+
+    AbstractBlockPackedWriter(IndexOutput indexOutput, int blockSize)
+    {
+        checkBlockSize(blockSize, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE);
+        this.indexOutput = indexOutput;
+        this.blockMetaWriter = new RAMIndexOutput("BlockPackedMeta");
+        blockValues = new long[blockSize];
+    }
+
+
+    /**
+     * Append a new long.
+     */
+    public void add(long l) throws IOException
+    {
+        checkNotFinished();
+        if (blockIndex == blockValues.length)
+        {
+            flush();
+        }
+        blockValues[blockIndex++] = l;
+    }
+
+    /**
+     * Flush all buffered data to disk. This instance is not usable anymore
+     * after this method has been called.
+     *
+     * @return a file offset to the block metadata
+     */
+    public long finish() throws IOException
+    {
+        checkNotFinished();
+        if (blockIndex > 0)
+        {
+            flush();
+        }
+        final long fp = indexOutput.getFilePointer();
+        blockMetaWriter.writeTo(indexOutput);
+        finished = true;
+        return fp;
+    }
+
+    protected abstract void flushBlock() throws IOException;
+
+    void writeValues(int numValues, int bitsPerValue) throws IOException
+    {
+        final DirectWriter writer = DirectWriter.getInstance(indexOutput, numValues, bitsPerValue);
+        for (int i = 0; i < numValues; ++i)
+        {
+            writer.add(blockValues[i]);
+        }
+        writer.finish();
+    }
+
+    void writeVLong(IndexOutput out, long i) throws IOException
+    {
+        int k = 0;
+        while ((i & ~0x7FL) != 0L && k++ < 8)
+        {
+            out.writeByte((byte) ((i & 0x7FL) | 0x80L));
+            i >>>= 7;
+        }
+        out.writeByte((byte) i);
+    }
+
+    private void flush() throws IOException
+    {
+        flushBlock();
+        blockIndex = 0;
+    }
+
+    private void checkNotFinished()
+    {
+        if (finished)
+        {
+            throw new IllegalStateException(String.format("[%s] Writer already finished!", indexOutput.getName()));
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/BlockPackedReader.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/BlockPackedReader.java
new file mode 100644
index 0000000000..fa42b25fb8
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/BlockPackedReader.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.bitpack;
+
+import java.io.IOException;
+
+import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
+import org.apache.cassandra.index.sai.disk.io.IndexInputReader;
+import org.apache.cassandra.index.sai.disk.v1.DirectReaders;
+import org.apache.cassandra.index.sai.disk.v1.LongArray;
+import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.store.IndexInput;
+
+import static org.apache.cassandra.index.sai.disk.v1.SAICodecUtils.checkBlockSize;
+import static org.apache.cassandra.index.sai.disk.v1.SAICodecUtils.numBlocks;
+import static org.apache.cassandra.index.sai.disk.v1.SAICodecUtils.readVLong;
+import static org.apache.lucene.util.BitUtil.zigZagDecode;
+
+/**
+ * Provides non-blocking, random access to a stream written with {@link BlockPackedWriter}.
+ */
+public class BlockPackedReader implements LongArray.Factory
+{
+    private final FileHandle file;
+    private final int blockShift;
+    private final int blockMask;
+    private final long valueCount;
+    private final byte[] blockBitsPerValue;
+    private final long[] blockOffsets;
+    private final long[] minValues;
+
+    public BlockPackedReader(FileHandle file, NumericValuesMeta meta) throws IOException
+    {
+        this.file = file;
+
+        this.valueCount = meta.valueCount;
+
+        blockShift = checkBlockSize(meta.blockSize, AbstractBlockPackedWriter.MIN_BLOCK_SIZE, AbstractBlockPackedWriter.MAX_BLOCK_SIZE);
+        blockMask = meta.blockSize - 1;
+        int numBlocks = numBlocks(valueCount, meta.blockSize);
+        blockBitsPerValue = new byte[numBlocks];
+        blockOffsets = new long[numBlocks];
+        minValues = new long[numBlocks];
+
+        try (RandomAccessReader reader = this.file.createReader();
+             IndexInputReader in = IndexInputReader.create(reader))
+        {
+            SAICodecUtils.validate(in);
+            in.seek(meta.blockMetaOffset);
+
+            for (int i = 0; i < numBlocks; ++i)
+            {
+                final int token = in.readByte() & 0xFF;
+                final int bitsPerValue = token >>> BlockPackedWriter.BPV_SHIFT;
+                if (!DirectReaders.SUPPORTED_BITS_PER_VALUE.contains(bitsPerValue))
+                {
+                    throw new CorruptIndexException(String.format("Block %d is corrupted. Bits per value is %d. Supported values are %s.",
+                                                                  i, bitsPerValue, DirectReaders.SUPPORTED_BITS_PER_VALUE_STRING), in);
+                }
+                if ((token & BlockPackedWriter.MIN_VALUE_EQUALS_0) == 0)
+                {
+                    long val = zigZagDecode(1L + readVLong(in));
+                    minValues[i] = val;
+                }
+                else
+                {
+                    minValues[i] = 0L;
+                }
+
+                blockBitsPerValue[i] = (byte) bitsPerValue;
+
+                if (bitsPerValue > 0)
+                {
+                    blockOffsets[i] = in.readVLong();
+                }
+                else
+                {
+                    blockOffsets[i] = -1;
+                }
+            }
+        }
+    }
+
+    @Override
+    @SuppressWarnings({"resource", "RedundantSuppression"})
+    public LongArray open()
+    {
+        IndexInput indexInput = IndexFileUtils.instance.openInput(file);
+        return new AbstractBlockPackedReader(indexInput, blockBitsPerValue, blockShift, blockMask, valueCount)
+        {
+            @Override
+            protected long blockOffsetAt(int block)
+            {
+                return blockOffsets[block];
+            }
+
+            @Override
+            long delta(int block, int idx)
+            {
+                return minValues[block];
+            }
+
+            @Override
+            public void close() throws IOException
+            {
+                indexInput.close();
+            }
+        };
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/BlockPackedWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/BlockPackedWriter.java
new file mode 100644
index 0000000000..9dcc29c6ab
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/BlockPackedWriter.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.bitpack;
+
+import java.io.IOException;
+
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.packed.DirectWriter;
+
+import static org.apache.lucene.util.BitUtil.zigZagEncode;
+
+/**
+ * A writer for large sequences of longs.
+ *
+ * Modified copy of {@link org.apache.lucene.util.packed.BlockPackedWriter} to use {@link DirectWriter}
+ * for optimised reads that doesn't require seeking through the whole file to open a thread-exclusive reader.
+ */
+public class BlockPackedWriter extends AbstractBlockPackedWriter
+{
+    static final int BPV_SHIFT = 1;
+    static final int MIN_VALUE_EQUALS_0 = 1;
+
+    public BlockPackedWriter(IndexOutput out, int blockSize)
+    {
+        super(out, blockSize);
+    }
+
+    @Override
+    protected void flushBlock() throws IOException
+    {
+        long min = Long.MAX_VALUE, max = Long.MIN_VALUE;
+        for (int i = 0; i < blockIndex; ++i)
+        {
+            min = Math.min(blockValues[i], min);
+            max = Math.max(blockValues[i], max);
+        }
+
+        long delta = max - min;
+        int bitsRequired = delta == 0 ? 0 : DirectWriter.unsignedBitsRequired(delta);
+
+        int shiftedBitsRequired = (bitsRequired << BPV_SHIFT) | (min == 0 ? MIN_VALUE_EQUALS_0 : 0);
+        blockMetaWriter.writeByte((byte) shiftedBitsRequired);
+
+        if (min != 0)
+        {
+            writeVLong(blockMetaWriter, zigZagEncode(min) - 1);
+        }
+
+        if (bitsRequired > 0)
+        {
+            if (min != 0)
+            {
+                for (int i = 0; i < blockIndex; ++i)
+                {
+                    blockValues[i] -= min;
+                }
+            }
+            blockMetaWriter.writeVLong(indexOutput.getFilePointer());
+            writeValues(blockIndex, bitsRequired);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/MonotonicBlockPackedReader.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/MonotonicBlockPackedReader.java
new file mode 100644
index 0000000000..872a44e02f
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/MonotonicBlockPackedReader.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.bitpack;
+
+import java.io.IOException;
+
+import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
+import org.apache.cassandra.index.sai.disk.io.IndexInputReader;
+import org.apache.cassandra.index.sai.disk.v1.DirectReaders;
+import org.apache.cassandra.index.sai.disk.v1.LongArray;
+import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.packed.PackedInts;
+import org.apache.lucene.util.packed.PackedLongValues;
+
+import static org.apache.cassandra.index.sai.disk.v1.SAICodecUtils.checkBlockSize;
+import static org.apache.cassandra.index.sai.disk.v1.SAICodecUtils.numBlocks;
+
+/**
+ * Provides non-blocking, random access to a stream written with {@link MonotonicBlockPackedWriter}.
+ */
+public class MonotonicBlockPackedReader implements LongArray.Factory
+{
+    private final FileHandle file;
+    private final int blockShift;
+    private final int blockMask;
+    private final long valueCount;
+    private final byte[] blockBitsPerValue;
+    private final PackedLongValues blockOffsets;
+    private final PackedLongValues minValues;
+    private final float[] averages;
+
+    public MonotonicBlockPackedReader(FileHandle file, NumericValuesMeta meta) throws IOException
+    {
+        this.valueCount = meta.valueCount;
+        blockShift = checkBlockSize(meta.blockSize, AbstractBlockPackedWriter.MIN_BLOCK_SIZE, AbstractBlockPackedWriter.MAX_BLOCK_SIZE);
+        blockMask = meta.blockSize - 1;
+        int numBlocks = numBlocks(valueCount, meta.blockSize);
+        PackedLongValues.Builder minValuesBuilder = PackedLongValues.monotonicBuilder(PackedInts.COMPACT);
+        PackedLongValues.Builder blockOffsetsBuilder = PackedLongValues.monotonicBuilder(PackedInts.COMPACT);
+        averages = new float[numBlocks];
+        blockBitsPerValue = new byte[numBlocks];
+        this.file = file;
+
+        try (RandomAccessReader reader = this.file.createReader();
+             IndexInputReader in = IndexInputReader.create(reader))
+        {
+            SAICodecUtils.validate(in);
+
+            in.seek(meta.blockMetaOffset);
+            for (int i = 0; i < numBlocks; ++i)
+            {
+                minValuesBuilder.add(in.readZLong());
+                averages[i] = Float.intBitsToFloat(in.readInt());
+                final int bitsPerValue = in.readVInt();
+                if (!DirectReaders.SUPPORTED_BITS_PER_VALUE.contains(bitsPerValue))
+                {
+                    throw new CorruptIndexException(String.format("Block %d is corrupted. Bits per value is %d. Supported values are %s.",
+                                                                  i, bitsPerValue, DirectReaders.SUPPORTED_BITS_PER_VALUE_STRING), in);
+                }
+                blockBitsPerValue[i] = (byte) bitsPerValue;
+                // when bitsPerValue is 0, block offset won't be used
+                blockOffsetsBuilder.add(bitsPerValue == 0 ? -1 : in.readVLong());
+            }
+        }
+
+        blockOffsets = blockOffsetsBuilder.build();
+        minValues = minValuesBuilder.build();
+    }
+
+    @Override
+    @SuppressWarnings({"resource", "RedundantSuppression"})
+    public LongArray open()
+    {
+        final IndexInput indexInput = IndexFileUtils.instance.openInput(file);
+        return new AbstractBlockPackedReader(indexInput, blockBitsPerValue, blockShift, blockMask, valueCount)
+        {
+            @Override
+            long delta(int block, int idx)
+            {
+                return expected(minValues.get(block), averages[block], idx);
+            }
+
+            @Override
+            public void close() throws IOException
+            {
+                indexInput.close();
+            }
+
+            @Override
+            protected long blockOffsetAt(int block)
+            {
+                return blockOffsets.get(block);
+            }
+        };
+    }
+
+    public static long expected(long origin, float average, int index)
+    {
+        return origin + (long)(average * index);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/MonotonicBlockPackedWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/MonotonicBlockPackedWriter.java
new file mode 100644
index 0000000000..5845aabc77
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/MonotonicBlockPackedWriter.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.bitpack;
+
+import java.io.IOException;
+
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.packed.DirectWriter;
+
+/**
+ * A writer for large monotonically increasing sequences of positive longs.
+ *
+ * The writer is optimised for monotonic sequences and stores values as a series of deltas
+ * from an expected value. The expected value is calculated from the minimum value in the block and the average
+ * delta for the block. This means that stored values are generally smaller and can be packed
+ * into a smaller number of bits, allowing for larger block sizes.
+ *
+ * Modified copy of {@link org.apache.lucene.util.packed.MonotonicBlockPackedWriter} to use {@link DirectWriter} for
+ * optimised reads that doesn't require seeking through the whole file to open a thread-exclusive reader.
+ */
+public class MonotonicBlockPackedWriter extends AbstractBlockPackedWriter
+{
+    public MonotonicBlockPackedWriter(IndexOutput out, int blockSize)
+    {
+        super(out, blockSize);
+    }
+
+    @Override
+    public void add(long l) throws IOException
+    {
+        assert l >= 0;
+        super.add(l);
+    }
+
+    @Override
+    protected void flushBlock() throws IOException
+    {
+        final float averageDelta = blockIndex == 1 ? 0f : (float) (blockValues[blockIndex - 1] - blockValues[0]) / (blockIndex - 1);
+        long minimumValue = blockValues[0];
+        // adjust minimumValue so that all deltas will be positive
+        for (int index = 1; index < blockIndex; ++index)
+        {
+            long actual = blockValues[index];
+            long expected = MonotonicBlockPackedReader.expected(minimumValue, averageDelta, index);
+            if (expected > actual)
+            {
+                minimumValue -= (expected - actual);
+            }
+        }
+
+        long maxDelta = 0;
+        for (int i = 0; i < blockIndex; ++i)
+        {
+            blockValues[i] = blockValues[i] - MonotonicBlockPackedReader.expected(minimumValue, averageDelta, i);
+            maxDelta = Math.max(maxDelta, blockValues[i]);
+        }
+
+        blockMetaWriter.writeZLong(minimumValue);
+        blockMetaWriter.writeInt(Float.floatToIntBits(averageDelta));
+        if (maxDelta == 0)
+        {
+            blockMetaWriter.writeVInt(0);
+        }
+        else
+        {
+            final int bitsRequired = DirectWriter.bitsRequired(maxDelta);
+            blockMetaWriter.writeVInt(bitsRequired);
+            blockMetaWriter.writeVLong(indexOutput.getFilePointer());
+            writeValues(blockIndex, bitsRequired);
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/virtual/StorageAttachedIndexTables.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/NumericValuesMeta.java
similarity index 52%
copy from src/java/org/apache/cassandra/index/sai/virtual/StorageAttachedIndexTables.java
copy to src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/NumericValuesMeta.java
index 0d4f12678f..9d52150f5f 100644
--- a/src/java/org/apache/cassandra/index/sai/virtual/StorageAttachedIndexTables.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/NumericValuesMeta.java
@@ -15,21 +15,30 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+package org.apache.cassandra.index.sai.disk.v1.bitpack;
 
-package org.apache.cassandra.index.sai.virtual;
+import java.io.IOException;
 
-import java.util.Collection;
-import java.util.Collections;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
 
-import org.apache.cassandra.db.virtual.VirtualTable;
-
-public class StorageAttachedIndexTables
+public class NumericValuesMeta
 {
-    private StorageAttachedIndexTables()
-    {}
+    public final long valueCount;
+    public final int blockSize;
+    public final long blockMetaOffset;
+
+    public NumericValuesMeta(IndexInput input) throws IOException
+    {
+        valueCount = input.readLong();
+        blockSize = input.readInt();
+        blockMetaOffset = input.readVLong();
+    }
 
-    public static Collection<VirtualTable> getAll(String keyspace)
+    public static void write(IndexOutput out, long valueCount, int blockSize, long blockMetaOffset) throws IOException
     {
-        return Collections.singletonList(new IndexesSystemView(keyspace));
+        out.writeLong(valueCount);
+        out.writeInt(blockSize);
+        out.writeVLong(blockMetaOffset);
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/NumericValuesWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/NumericValuesWriter.java
new file mode 100644
index 0000000000..461c15b7dd
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/bitpack/NumericValuesWriter.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.bitpack;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.v1.MetadataWriter;
+import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
+import org.apache.lucene.store.IndexOutput;
+
+@NotThreadSafe
+public class NumericValuesWriter implements Closeable
+{
+    public static final int MONOTONIC_BLOCK_SIZE = 16384;
+    public static final int BLOCK_SIZE = 128;
+
+    private final IndexOutput output;
+    private final AbstractBlockPackedWriter writer;
+    private final MetadataWriter metadataWriter;
+    private final String componentName;
+    private final int blockSize;
+    private long count = 0;
+
+    public NumericValuesWriter(String componentName,
+                               IndexOutput indexOutput,
+                               MetadataWriter metadataWriter,
+                               boolean monotonic) throws IOException
+    {
+        this(componentName, indexOutput, metadataWriter, monotonic, monotonic ? MONOTONIC_BLOCK_SIZE : BLOCK_SIZE);
+    }
+
+    public NumericValuesWriter(IndexDescriptor indexDescriptor,
+                               IndexComponent component,
+                               MetadataWriter metadataWriter,
+                               boolean monotonic,
+                               int blockSize) throws IOException
+    {
+        this(indexDescriptor.componentName(component),
+             indexDescriptor.openPerSSTableOutput(component),
+             metadataWriter,
+             monotonic,
+             blockSize);
+    }
+
+    private NumericValuesWriter(String componentName,
+                                IndexOutput indexOutput,
+                                MetadataWriter metadataWriter,
+                                boolean monotonic, int blockSize) throws IOException
+    {
+        SAICodecUtils.writeHeader(indexOutput);
+        this.writer = monotonic ? new MonotonicBlockPackedWriter(indexOutput, blockSize)
+                                : new BlockPackedWriter(indexOutput, blockSize);
+        this.output = indexOutput;
+        this.componentName = componentName;
+        this.metadataWriter = metadataWriter;
+        this.blockSize = blockSize;
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        try (IndexOutput o = metadataWriter.builder(componentName))
+        {
+            long fp = writer.finish();
+            SAICodecUtils.writeFooter(output);
+
+            NumericValuesMeta.write(o, count, blockSize, fp);
+        }
+        finally
+        {
+            output.close();
+        }
+    }
+
+    public void add(long value) throws IOException
+    {
+        writer.add(value);
+        count++;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/MergePostingList.java b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/MergePostingList.java
new file mode 100644
index 0000000000..e20986dd71
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/MergePostingList.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.postings;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.PriorityQueue;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.cassandra.index.sai.postings.PeekablePostingList;
+import org.apache.cassandra.index.sai.postings.PostingList;
+import org.apache.cassandra.io.util.FileUtils;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+/**
+ * Merges multiple {@link PostingList} which individually contain unique items into a single list.
+ * While the individual lists contain unique items, there can be duplicate items between lists so
+ * the class also checks for duplicates and only returns unique items in sorted order.
+ */
+@NotThreadSafe
+public class MergePostingList implements PostingList
+{
+    private final PriorityQueue<PeekablePostingList> postingLists;
+    private final List<PeekablePostingList> temp;
+    private final Closeable onClose;
+    private final long size;
+    private long lastRowId = -1;
+
+    private MergePostingList(PriorityQueue<PeekablePostingList> postingLists, Closeable onClose)
+    {
+        this.temp = new ArrayList<>(postingLists.size());
+        this.onClose = onClose;
+        this.postingLists = postingLists;
+        long totalPostings = 0;
+        for (PostingList postingList : postingLists)
+        {
+            totalPostings += postingList.size();
+        }
+        this.size = totalPostings;
+    }
+
+    public static PostingList merge(PriorityQueue<PeekablePostingList> postings, Closeable onClose)
+    {
+        checkArgument(!postings.isEmpty(), "Cannot merge an empty queue of posting lists");
+        return postings.size() > 1 ? new MergePostingList(postings, onClose) : postings.poll();
+    }
+
+    public static PostingList merge(PriorityQueue<PeekablePostingList> postings)
+    {
+        return merge(postings, () -> FileUtils.close(postings));
+    }
+
+    @SuppressWarnings({"resource", "RedundantSuppression"})
+    @Override
+    public long nextPosting() throws IOException
+    {
+        while (!postingLists.isEmpty())
+        {
+            PeekablePostingList head = postingLists.poll();
+            long next = head.nextPosting();
+
+            if (next == END_OF_STREAM)
+            {
+                // skip current posting list
+                continue;
+            }
+
+            if (next > lastRowId)
+            {
+                lastRowId = next;
+                postingLists.add(head);
+                return next;
+            }
+            else if (next == lastRowId)
+            {
+                postingLists.add(head);
+            }
+        }
+
+        return PostingList.END_OF_STREAM;
+    }
+
+    @SuppressWarnings({"resource", "RedundantSuppression"})
+    @Override
+    public long advance(long targetRowID) throws IOException
+    {
+        temp.clear();
+
+        while (!postingLists.isEmpty())
+        {
+            PeekablePostingList peekable = postingLists.poll();
+            peekable.advanceWithoutConsuming(targetRowID);
+            if (peekable.peek() != PostingList.END_OF_STREAM)
+                temp.add(peekable);
+        }
+        postingLists.addAll(temp);
+
+        return nextPosting();
+    }
+
+    @Override
+    public long size()
+    {
+        return size;
+    }
+
+    @Override
+    public void close()
+    {
+        FileUtils.closeQuietly(onClose);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingListRangeIterator.java b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingListRangeIterator.java
new file mode 100644
index 0000000000..354663a30d
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingListRangeIterator.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.postings;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import com.google.common.base.Stopwatch;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.exceptions.QueryCancelledException;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.disk.v1.segment.IndexSegmentSearcherContext;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.postings.PostingList;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.Throwables;
+
+/**
+ * A key iterator based on a  {@link PostingList} derived from a single index segment.
+ *
+ * <ol>
+ *   <li> fetch next segment row id from posting list or skip to specific segment row id if {@link #skipTo(PrimaryKey)} is called </li>
+ *   <li> add {@link IndexSegmentSearcherContext#segmentRowIdOffset} to obtain the sstable row id </li>
+ *   <li> produce a {@link PrimaryKey} from {@link PrimaryKeyMap#primaryKeyFromRowId(long)} which is used
+ *       to avoid fetching duplicated keys due to partition-level indexing on wide partition schema.
+ *       <br/>
+ *       Note: in order to reduce disk access in multi-index query, partition keys will only be fetched for intersected tokens
+ *       in {@link org.apache.cassandra.index.sai.plan.StorageAttachedIndexSearcher}.
+ *  </li>
+ * </ol>
+ *
+ */
+
+@NotThreadSafe
+public class PostingListRangeIterator extends KeyRangeIterator
+{
+    private static final Logger logger = LoggerFactory.getLogger(PostingListRangeIterator.class);
+
+    private final Stopwatch timeToExhaust = Stopwatch.createStarted();
+    private final QueryContext queryContext;
+
+    private final PostingList postingList;
+    private final IndexContext indexContext;
+    private final PrimaryKeyMap primaryKeyMap;
+    private final IndexSegmentSearcherContext searcherContext;
+
+    private boolean needsSkipping = false;
+    private PrimaryKey skipToToken = null;
+
+
+    /**
+     * Create a direct PostingListRangeIterator where the underlying PostingList is materialised
+     * immediately so the posting list size can be used.
+     */
+    public PostingListRangeIterator(IndexContext indexContext,
+                                    PrimaryKeyMap primaryKeyMap,
+                                    IndexSegmentSearcherContext searcherContext)
+    {
+        super(searcherContext.minimumKey, searcherContext.maximumKey, searcherContext.count());
+
+        this.indexContext = indexContext;
+        this.primaryKeyMap = primaryKeyMap;
+        this.postingList = searcherContext.postingList;
+        this.searcherContext = searcherContext;
+        this.queryContext = searcherContext.context;
+    }
+
+    @Override
+    protected void performSkipTo(PrimaryKey nextKey)
+    {
+        if (skipToToken != null && skipToToken.compareTo(nextKey) >= 0)
+            return;
+
+        skipToToken = nextKey;
+        needsSkipping = true;
+    }
+
+    @Override
+    protected PrimaryKey computeNext()
+    {
+        try
+        {
+            queryContext.checkpoint();
+
+            // just end the iterator if we don't have a postingList or current segment is skipped
+            if (exhausted())
+                return endOfData();
+
+            long rowId = getNextRowId();
+            if (rowId == PostingList.END_OF_STREAM)
+                return endOfData();
+
+            return primaryKeyMap.primaryKeyFromRowId(rowId);
+        }
+        catch (Throwable t)
+        {
+            if (!(t instanceof QueryCancelledException))
+                logger.error(indexContext.logMessage("Unable to provide next token!"), t);
+
+            throw Throwables.cleaned(t);
+        }
+    }
+
+    @Override
+    public void close()
+    {
+        if (logger.isTraceEnabled())
+        {
+            final long exhaustedInMills = timeToExhaust.stop().elapsed(TimeUnit.MILLISECONDS);
+            logger.trace(indexContext.logMessage("PostingListRangeIterator exhausted after {} ms"), exhaustedInMills);
+        }
+
+        FileUtils.closeQuietly(Arrays.asList(postingList, primaryKeyMap));
+    }
+
+    private boolean exhausted()
+    {
+        return needsSkipping && skipToToken.compareTo(getMaximum()) > 0;
+    }
+
+    /**
+     * reads the next sstable row ID from the underlying posting list, potentially skipping to get there.
+     */
+    private long getNextRowId() throws IOException
+    {
+        long segmentRowId;
+        if (needsSkipping)
+        {
+            long targetRowID = primaryKeyMap.rowIdFromPrimaryKey(skipToToken);
+            // skipToToken is larger than max token in token file
+            if (targetRowID < 0)
+            {
+                return PostingList.END_OF_STREAM;
+            }
+
+            segmentRowId = postingList.advance(targetRowID - searcherContext.segmentRowIdOffset);
+
+            needsSkipping = false;
+        }
+        else
+        {
+            segmentRowId = postingList.nextPosting();
+        }
+
+        return segmentRowId != PostingList.END_OF_STREAM
+               ? segmentRowId + searcherContext.segmentRowIdOffset
+               : PostingList.END_OF_STREAM;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingsReader.java b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingsReader.java
new file mode 100644
index 0000000000..de82ebd01c
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingsReader.java
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.postings;
+
+
+import java.io.IOException;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.index.sai.postings.OrdinalPostingList;
+import org.apache.cassandra.index.sai.postings.PostingList;
+import org.apache.cassandra.index.sai.disk.v1.DirectReaders;
+import org.apache.cassandra.index.sai.disk.v1.LongArray;
+import org.apache.cassandra.index.sai.metrics.QueryEventListener;
+import org.apache.cassandra.index.sai.disk.io.SeekingRandomAccessInput;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.RandomAccessInput;
+
+
+/**
+ * Reads, decompresses and decodes postings lists written by {@link PostingsWriter}.
+ *
+ * Holds exactly one posting block in memory at a time. Does binary search over skip table to find a postings block to
+ * load.
+ */
+@NotThreadSafe
+public class PostingsReader implements OrdinalPostingList
+{
+    private final IndexInput input;
+    private final SeekingRandomAccessInput seekingInput;
+    private final QueryEventListener.PostingListEventListener listener;
+    private final BlocksSummary summary;
+
+    // Current block index
+    private int blockIndex;
+    // Current posting index within block
+    private int postingIndex;
+    private long totalPostingsRead;
+    private long actualPosting;
+
+    private long currentPosition;
+    private DirectReaders.Reader currentFoRValues;
+    private long postingsDecoded = 0;
+
+    @VisibleForTesting
+    public PostingsReader(IndexInput input, long summaryOffset, QueryEventListener.PostingListEventListener listener) throws IOException
+    {
+        this(input, new BlocksSummary(input, summaryOffset), listener);
+    }
+
+    public PostingsReader(IndexInput input, BlocksSummary summary, QueryEventListener.PostingListEventListener listener) throws IOException
+    {
+        this.input = input;
+        this.seekingInput = new SeekingRandomAccessInput(input);
+        this.listener = listener;
+        this.summary = summary;
+
+        reBuffer();
+    }
+
+    @Override
+    public long getOrdinal()
+    {
+        return totalPostingsRead;
+    }
+
+    public static class BlocksSummary
+    {
+        private final IndexInput input;
+        final int blockSize;
+        final int numPostings;
+        final LongArray offsets;
+        final LongArray maxValues;
+
+        public BlocksSummary(IndexInput input, long offset) throws IOException
+        {
+            this.input = input;
+            input.seek(offset);
+            this.blockSize = input.readVInt();
+            //TODO This should need to change because we can potentially end up with postings of more than Integer.MAX_VALUE?
+            this.numPostings = input.readVInt();
+
+            SeekingRandomAccessInput randomAccessInput = new SeekingRandomAccessInput(input);
+            int numBlocks = input.readVInt();
+            long maxBlockValuesLength = input.readVLong();
+            long maxBlockValuesOffset = input.getFilePointer() + maxBlockValuesLength;
+
+            byte offsetBitsPerValue = input.readByte();
+            if (!DirectReaders.SUPPORTED_BITS_PER_VALUE.contains((int)offsetBitsPerValue))
+            {
+                throw new CorruptIndexException(
+                        String.format("Postings list header is corrupted: Bits per value for block offsets is %s. Supported values are %s.",
+                                      offsetBitsPerValue, DirectReaders.SUPPORTED_BITS_PER_VALUE_STRING), input);
+            }
+            this.offsets = new LongArrayReader(randomAccessInput, DirectReaders.getReaderForBitsPerValue(offsetBitsPerValue), input.getFilePointer(), numBlocks);
+
+            input.seek(maxBlockValuesOffset);
+            byte valuesBitsPerValue = input.readByte();
+            if (!DirectReaders.SUPPORTED_BITS_PER_VALUE.contains((int)valuesBitsPerValue))
+            {
+                throw new CorruptIndexException(
+                String.format("Postings list header is corrupted: Bits per value for value samples is %s. Supported values are %s.",
+                              valuesBitsPerValue, DirectReaders.SUPPORTED_BITS_PER_VALUE_STRING), input);
+            }
+            this.maxValues = new LongArrayReader(randomAccessInput, DirectReaders.getReaderForBitsPerValue(valuesBitsPerValue), input.getFilePointer(), numBlocks);
+        }
+
+        void close()
+        {
+            FileUtils.closeQuietly(input);
+        }
+
+        private static class LongArrayReader implements LongArray
+        {
+            private final RandomAccessInput input;
+            private final DirectReaders.Reader reader;
+            private final long offset;
+            private final int length;
+
+            private LongArrayReader(RandomAccessInput input, DirectReaders.Reader reader, long offset, int length)
+            {
+                this.input = input;
+                this.reader = reader;
+                this.offset = offset;
+                this.length = length;
+            }
+
+            @Override
+            public long get(long idx)
+            {
+                return reader.get(input, offset, idx);
+            }
+
+            @Override
+            public long length()
+            {
+                return length;
+            }
+        }
+    }
+
+    @Override
+    public void close()
+    {
+        listener.postingDecoded(postingsDecoded);
+        FileUtils.closeQuietly(input);
+        summary.close();
+    }
+
+    @Override
+    public long size()
+    {
+        return summary.numPostings;
+    }
+
+    /**
+     * Advances to the first row ID beyond the current that is greater than or equal to the
+     * target, and returns that row ID. Exhausts the iterator and returns {@link #END_OF_STREAM} if
+     * the target is greater than the highest row ID.
+     *
+     * Does binary search over the skip table to find the next block to load into memory.
+     *
+     * Note: Callers must use the return value of this method before calling {@link #nextPosting()}, as calling
+     * that method will return the next posting, not the one to which we have just advanced.
+     *
+     * @param targetRowID target row ID to advance to
+     *
+     * @return first segment row ID which is >= the target row ID or {@link PostingList#END_OF_STREAM} if one does not exist
+     */
+    @Override
+    public long advance(long targetRowID) throws IOException
+    {
+        listener.onAdvance();
+        int block = binarySearchBlocks(targetRowID);
+
+        if (block < 0)
+        {
+            block = -block - 1;
+        }
+
+        if (blockIndex == block + 1)
+        {
+            // we're in the same block, just iterate through
+            return slowAdvance(targetRowID);
+        }
+        assert block > 0;
+        // Even if there was an exact match, block might contain duplicates.
+        // We iterate to the target token from the beginning.
+        lastPosInBlock(block - 1);
+        return slowAdvance(targetRowID);
+    }
+
+    private long slowAdvance(long targetRowID) throws IOException
+    {
+        while (totalPostingsRead < summary.numPostings)
+        {
+            long segmentRowId = peekNext();
+
+            advanceOnePosition(segmentRowId);
+
+            if (segmentRowId >= targetRowID)
+            {
+                return segmentRowId;
+            }
+        }
+        return END_OF_STREAM;
+    }
+
+    // Perform a binary search of the blocks to the find the block index
+    // containing the targetRowID, or, in the case of a duplicate value
+    // crossing blocks, the preceeding block index
+    private int binarySearchBlocks(long targetRowID)
+    {
+        int lowBlockIndex = blockIndex - 1;
+        int highBlockIndex = Math.toIntExact(summary.maxValues.length()) - 1;
+
+        // in current block
+        if (lowBlockIndex <= highBlockIndex && targetRowID <= summary.maxValues.get(lowBlockIndex))
+            return lowBlockIndex;
+
+        while (lowBlockIndex <= highBlockIndex)
+        {
+            int midBlockIndex = lowBlockIndex + ((highBlockIndex - lowBlockIndex) >> 1) ;
+
+            long maxValueOfMidBlock = summary.maxValues.get(midBlockIndex);
+
+            if (maxValueOfMidBlock < targetRowID)
+            {
+                lowBlockIndex = midBlockIndex + 1;
+            }
+            else if (maxValueOfMidBlock > targetRowID)
+            {
+                highBlockIndex = midBlockIndex - 1;
+            }
+            else
+            {
+                // At this point the maximum value of the midway block matches our target.
+                //
+                // This following check is to see if we have a duplicate value in the last entry of the
+                // preceeding block. This check is only going to be successful if the entire current
+                // block is full of duplicates.
+                if (midBlockIndex > 0 && summary.maxValues.get(midBlockIndex - 1) == targetRowID)
+                {
+                    // there is a duplicate in the preceeding block so restrict search to finish
+                    // at that block
+                    highBlockIndex = midBlockIndex - 1;
+                }
+                else
+                {
+                    // no duplicates
+                    return midBlockIndex;
+                }
+            }
+        }
+        return -(lowBlockIndex + 1);  // target not found
+    }
+
+    private void lastPosInBlock(int block)
+    {
+        // blockMaxValues is integer only
+        actualPosting = summary.maxValues.get(block);
+        //upper bound, since we might've advanced to the last block, but upper bound is enough
+        totalPostingsRead += (summary.blockSize - postingIndex) + (block - blockIndex + 1) * (long)summary.blockSize;
+
+        blockIndex = block + 1;
+        postingIndex = summary.blockSize;
+    }
+
+    @Override
+    public long nextPosting() throws IOException
+    {
+        long next = peekNext();
+        if (next != END_OF_STREAM)
+        {
+            advanceOnePosition(next);
+        }
+        return next;
+    }
+
+    private long peekNext() throws IOException
+    {
+        if (totalPostingsRead >= summary.numPostings)
+        {
+            return END_OF_STREAM;
+        }
+        if (postingIndex == summary.blockSize)
+        {
+            reBuffer();
+        }
+
+        return actualPosting + nextFoRValue();
+    }
+
+    private int nextFoRValue()
+    {
+        // currentFoRValues is null when the all the values in the block are the same
+        if (currentFoRValues == null)
+        {
+            return 0;
+        }
+        else
+        {
+            long id = currentFoRValues.get(seekingInput, currentPosition, postingIndex);
+            postingsDecoded++;
+            return Math.toIntExact(id);
+        }
+    }
+
+    private void advanceOnePosition(long nextPosting)
+    {
+        actualPosting = nextPosting;
+        totalPostingsRead++;
+        postingIndex++;
+    }
+
+    private void reBuffer() throws IOException
+    {
+        long pointer = summary.offsets.get(blockIndex);
+
+        input.seek(pointer);
+
+        long left = summary.numPostings - totalPostingsRead;
+        assert left > 0;
+
+        readFoRBlock(input);
+
+        blockIndex++;
+        postingIndex = 0;
+    }
+
+    private void readFoRBlock(IndexInput in) throws IOException
+    {
+        if (blockIndex == 0)
+            actualPosting = in.readVLong();
+
+        byte bitsPerValue = in.readByte();
+
+        currentPosition = in.getFilePointer();
+
+        if (bitsPerValue == 0)
+        {
+            // currentFORValues is null when the all the values in the block are the same
+            currentFoRValues = null;
+            return;
+        }
+        currentFoRValues = DirectReaders.getReaderForBitsPerValue(bitsPerValue);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingsWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingsWriter.java
new file mode 100644
index 0000000000..d441942fc4
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/postings/PostingsWriter.java
@@ -0,0 +1,316 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.postings;
+
+
+import java.io.Closeable;
+import java.io.IOException;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.agrona.collections.LongArrayList;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.disk.v1.DirectReaders;
+import org.apache.cassandra.index.sai.postings.PostingList;
+import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.io.RAMIndexOutput;
+import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.packed.DirectWriter;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.lang.Math.max;
+import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.BLOCK_SIZE;
+
+/**
+ * Encodes, compresses and writes postings lists to disk.
+ *
+ * All postings in the posting list are delta encoded, then deltas are divided into blocks for compression.
+ * The deltas are based on the final value of the previous block. For the first block in the posting list
+ * the first value in the block is written as a VLong prior to block delta encodings.
+ * <p>
+ * In packed blocks, longs are encoded with the same bit width (FoR compression). The block size (i.e. number of
+ * longs inside block) is fixed (currently 128). Additionally blocks that are all the same value are encoded in an
+ * optimized way.
+ * </p>
+ * <p>
+ * In VLong blocks, longs are compressed with {@link DataOutput#writeVLong}. The block size is variable.
+ * </p>
+ *
+ * <p>
+ * Packed blocks are favoured, meaning when the postings are long enough, {@link PostingsWriter} will try
+ * to encode most data as a packed block. Take a term with 259 postings as an example, the first 256 postings are encoded
+ * as two packed blocks, while the remaining 3 are encoded as one VLong block.
+ * </p>
+ * <p>
+ * Each posting list ends with a block summary containing metadata and a skip table, written right after all postings
+ * blocks. Skip interval is the same as block size, and each skip entry points to the end of each block.
+ * Skip table consist of block offsets and last values of each block, compressed as two FoR blocks.
+ * </p>
+ *
+ * Visual representation of the disk format:
+ * <pre>
+ *
+ * +========+========================+=====+==============+===============+===============+=====+========================+========+
+ * | HEADER | POSTINGS LIST (TERM 1)                                                      | ... | POSTINGS LIST (TERM N) | FOOTER |
+ * +========+========================+=====+==============+===============+===============+=====+========================+========+
+ *          | FIRST VALUE| FOR BLOCK (1)| ... | FOR BLOCK (N)| BLOCK SUMMARY              |
+ *          +---------------------------+-----+--------------+---------------+------------+
+ *                                                           | BLOCK SIZE    |            |
+ *                                                           | LIST SIZE     | SKIP TABLE |
+ *                                                           +---------------+------------+
+ *                                                                           | BLOCKS POS.|
+ *                                                                           | MAX VALUES |
+ *                                                                           +------------+
+ *
+ *  </pre>
+ */
+@NotThreadSafe
+public class PostingsWriter implements Closeable
+{
+    private static final String POSTINGS_MUST_BE_SORTED_ERROR_MSG = "Postings must be sorted ascending, got [%s] after [%s]";
+
+    private final IndexOutput dataOutput;
+    private final int blockSize;
+    private final long[] deltaBuffer;
+    private final LongArrayList blockOffsets = new LongArrayList();
+    private final LongArrayList blockMaximumPostings = new LongArrayList();
+    private final RAMIndexOutput inMemoryOutput = new RAMIndexOutput("blockOffsets");
+
+    private final long startOffset;
+
+    private int bufferUpto;
+    private long firstPosting = Long.MIN_VALUE;
+    private long lastPosting = Long.MIN_VALUE;
+    private long maxDelta;
+    private long totalPostings;
+
+    public PostingsWriter(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException
+    {
+        this(indexDescriptor, indexContext, BLOCK_SIZE);
+    }
+
+    @VisibleForTesting
+    PostingsWriter(IndexDescriptor indexDescriptor, IndexContext indexContext, int blockSize) throws IOException
+    {
+        this(indexDescriptor.openPerIndexOutput(IndexComponent.POSTING_LISTS, indexContext, true), blockSize);
+    }
+
+    private PostingsWriter(IndexOutput dataOutput, int blockSize) throws IOException
+    {
+        this.blockSize = blockSize;
+        this.dataOutput = dataOutput;
+        startOffset = dataOutput.getFilePointer();
+        deltaBuffer = new long[blockSize];
+        SAICodecUtils.writeHeader(dataOutput);
+    }
+
+    /**
+     * @return current file pointer
+     */
+    public long getFilePointer()
+    {
+        return dataOutput.getFilePointer();
+    }
+
+    /**
+     * @return file pointer where index structure begins
+     */
+    public long getStartOffset()
+    {
+        return startOffset;
+    }
+
+    /**
+     * write footer to the postings
+     */
+    public void complete() throws IOException
+    {
+        SAICodecUtils.writeFooter(dataOutput);
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        dataOutput.close();
+    }
+
+    /**
+     * Encodes, compresses and flushes given posting list to disk.
+     *
+     * @param postings posting list to write to disk
+     *
+     * @return file offset to the summary block of this posting list
+     */
+    public long write(PostingList postings) throws IOException
+    {
+        checkArgument(postings != null, "Expected non-null posting list.");
+        checkArgument(postings.size() > 0, "Expected non-empty posting list.");
+
+        lastPosting = Long.MIN_VALUE;
+        resetBlockCounters();
+        blockOffsets.clear();
+        blockMaximumPostings.clear();
+
+        long posting;
+        // When postings list are merged, we don't know exact size, just an upper bound.
+        // We need to count how many postings we added to the block ourselves.
+        int size = 0;
+        while ((posting = postings.nextPosting()) != PostingList.END_OF_STREAM)
+        {
+            writePosting(posting);
+            size++;
+            totalPostings++;
+        }
+
+        assert size > 0 : "No postings were written";
+
+        finish();
+
+        final long summaryOffset = dataOutput.getFilePointer();
+        writeSummary(size);
+        return summaryOffset;
+    }
+
+    public long getTotalPostings()
+    {
+        return totalPostings;
+    }
+
+    private void writePosting(long posting) throws IOException
+    {
+        if (lastPosting == Long.MIN_VALUE)
+        {
+            firstPosting = posting;
+            deltaBuffer[bufferUpto++] = 0;
+        }
+        else
+        {
+            if (posting < lastPosting)
+                throw new IllegalArgumentException(String.format(POSTINGS_MUST_BE_SORTED_ERROR_MSG, posting, lastPosting));
+            long delta = posting - lastPosting;
+            maxDelta = max(maxDelta, delta);
+            deltaBuffer[bufferUpto++] = delta;
+        }
+        lastPosting = posting;
+
+        if (bufferUpto == blockSize)
+        {
+            addBlockToSkipTable();
+            writePostingsBlock();
+            resetBlockCounters();
+        }
+    }
+
+    private void finish() throws IOException
+    {
+        if (bufferUpto > 0)
+        {
+            addBlockToSkipTable();
+            writePostingsBlock();
+        }
+    }
+
+    private void resetBlockCounters()
+    {
+        firstPosting = Long.MIN_VALUE;
+        bufferUpto = 0;
+        maxDelta = 0;
+    }
+
+    private void addBlockToSkipTable()
+    {
+        blockOffsets.add(dataOutput.getFilePointer());
+        blockMaximumPostings.add(lastPosting);
+    }
+
+    private void writeSummary(int exactSize) throws IOException
+    {
+        dataOutput.writeVInt(blockSize);
+        dataOutput.writeVInt(exactSize);
+        writeSkipTable();
+    }
+
+    private void writeSkipTable() throws IOException
+    {
+        assert blockOffsets.size() == blockMaximumPostings.size();
+        dataOutput.writeVInt(blockOffsets.size());
+
+        // compressing offsets in memory first, to know the exact length (with padding)
+        inMemoryOutput.reset();
+
+        writeSortedFoRBlock(blockOffsets, inMemoryOutput);
+        dataOutput.writeVLong(inMemoryOutput.getFilePointer());
+        inMemoryOutput.writeTo(dataOutput);
+        writeSortedFoRBlock(blockMaximumPostings, dataOutput);
+    }
+
+    private void writePostingsBlock() throws IOException
+    {
+        final int bitsPerValue = maxDelta == 0 ? 0 : DirectWriter.unsignedBitsRequired(maxDelta);
+
+        assert DirectReaders.SUPPORTED_BITS_PER_VALUE.contains(bitsPerValue) :
+        "Unsupported bits per value of " + bitsPerValue + " bits. Supported values are: " + DirectReaders.SUPPORTED_BITS_PER_VALUE_STRING;
+
+        // If we have a first posting, indicating that this is the first block in the posting list
+        // then write it prior to the deltas.
+        if (firstPosting != Long.MIN_VALUE)
+            dataOutput.writeVLong(firstPosting);
+
+        dataOutput.writeByte((byte) bitsPerValue);
+        if (bitsPerValue > 0)
+        {
+            final DirectWriter writer = DirectWriter.getInstance(dataOutput, blockSize, bitsPerValue);
+            for (int index = 0; index < bufferUpto; ++index)
+            {
+                writer.add(deltaBuffer[index]);
+            }
+            if (bufferUpto < blockSize)
+            {
+                // Pad the rest of the block with 0, so we don't write invalid
+                // values from previous blocks
+                for (int index = bufferUpto; index < blockSize; index++)
+                {
+                    writer.add(0);
+                }
+            }
+            writer.finish();
+        }
+    }
+
+    private void writeSortedFoRBlock(LongArrayList values, IndexOutput output) throws IOException
+    {
+        final long maxValue = values.getLong(values.size() - 1);
+
+        assert values.size() > 0;
+        final int bitsPerValue = maxValue == 0 ? 0 : DirectWriter.unsignedBitsRequired(maxValue);
+        output.writeByte((byte) bitsPerValue);
+        if (bitsPerValue > 0)
+        {
+            final DirectWriter writer = DirectWriter.getInstance(output, values.size(), bitsPerValue);
+            for (int i = 0; i < values.size(); ++i)
+            {
+                writer.add(values.getLong(i));
+            }
+            writer.finish();
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/IndexSegmentSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/IndexSegmentSearcher.java
new file mode 100644
index 0000000000..bb22ff492b
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/IndexSegmentSearcher.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.segment;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles;
+import org.apache.cassandra.index.sai.disk.v1.postings.PostingListRangeIterator;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.postings.PeekablePostingList;
+import org.apache.cassandra.index.sai.postings.PostingList;
+
+/**
+ * Abstract reader for individual segments of an on-disk index.
+ *
+ * Accepts shared resources (token/offset file readers), and uses them to perform lookups against on-disk data
+ * structures.
+ */
+public abstract class IndexSegmentSearcher implements Closeable
+{
+    final PrimaryKeyMap.Factory primaryKeyMapFactory;
+    final PerColumnIndexFiles indexFiles;
+    final SegmentMetadata metadata;
+    final IndexContext indexContext;
+
+    IndexSegmentSearcher(PrimaryKeyMap.Factory primaryKeyMapFactory,
+                         PerColumnIndexFiles perIndexFiles,
+                         SegmentMetadata segmentMetadata,
+                         IndexContext indexContext)
+    {
+        this.primaryKeyMapFactory = primaryKeyMapFactory;
+        this.indexFiles = perIndexFiles;
+        this.metadata = segmentMetadata;
+        this.indexContext = indexContext;
+    }
+
+    public static IndexSegmentSearcher open(PrimaryKeyMap.Factory primaryKeyMapFactory,
+                                            PerColumnIndexFiles indexFiles,
+                                            SegmentMetadata segmentMetadata,
+                                            IndexContext indexContext) throws IOException
+    {
+        return new LiteralIndexSegmentSearcher(primaryKeyMapFactory, indexFiles, segmentMetadata, indexContext);
+    }
+
+    /**
+     * @return memory usage of underlying on-disk data structure
+     */
+    public abstract long indexFileCacheSize();
+
+    /**
+     * Search on-disk index synchronously.
+     *
+     * @param expression to filter on disk index
+     * @param queryContext to track per sstable cache and per query metrics
+     *
+     * @return {@link KeyRangeIterator} with matches for the given expression
+     */
+    public abstract KeyRangeIterator search(Expression expression, QueryContext queryContext) throws IOException;
+
+    KeyRangeIterator toIterator(PostingList postingList, QueryContext queryContext) throws IOException
+    {
+        if (postingList == null)
+            return KeyRangeIterator.empty();
+
+        IndexSegmentSearcherContext searcherContext = new IndexSegmentSearcherContext(metadata.minKey,
+                                                                                      metadata.maxKey,
+                                                                                      metadata.rowIdOffset,
+                                                                                      queryContext,
+                                                                                      PeekablePostingList.makePeekable(postingList));
+
+        return new PostingListRangeIterator(indexContext, primaryKeyMapFactory.newPerSSTablePrimaryKeyMap(), searcherContext);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/IndexSegmentSearcherContext.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/IndexSegmentSearcherContext.java
new file mode 100644
index 0000000000..5b3c59b0ec
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/IndexSegmentSearcherContext.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1.segment;
+
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.postings.PeekablePostingList;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+
+public class IndexSegmentSearcherContext
+{
+    public final QueryContext context;
+    public final PeekablePostingList postingList;
+
+    public final PrimaryKey minimumKey;
+    public final PrimaryKey maximumKey;
+    public final long segmentRowIdOffset;
+
+    public IndexSegmentSearcherContext(PrimaryKey minimumKey,
+                                       PrimaryKey maximumKey,
+                                       long segmentRowIdOffset,
+                                       QueryContext context,
+                                       PeekablePostingList postingList)
+    {
+        this.context = context;
+        this.postingList = postingList;
+
+        this.segmentRowIdOffset = segmentRowIdOffset;
+
+        this.minimumKey = minimumKey;
+
+        // use segment's metadata for the range iterator, may not be accurate, but should not matter to performance.
+        this.maximumKey = maximumKey;
+    }
+
+    public long count()
+    {
+        return postingList.size();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentSearcher.java
new file mode 100644
index 0000000000..fb45943934
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentSearcher.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sai.disk.v1.segment;
+
+import java.io.IOException;
+import java.util.Map;
+
+import com.google.common.base.MoreObjects;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.disk.format.IndexComponent;
+import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles;
+import org.apache.cassandra.index.sai.disk.v1.SAICodecUtils;
+import org.apache.cassandra.index.sai.metrics.MulticastQueryEventListeners;
+import org.apache.cassandra.index.sai.metrics.QueryEventListener;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.postings.PostingList;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+
+/**
+ * Executes {@link Expression}s against the trie-based terms dictionary for an individual index segment.
+ */
+public class LiteralIndexSegmentSearcher extends IndexSegmentSearcher
+{
+    private static final Logger logger = LoggerFactory.getLogger(LiteralIndexSegmentSearcher.class);
+
+    private final LiteralIndexSegmentTermsReader reader;
+    private final QueryEventListener.TrieIndexEventListener perColumnEventListener;
+
+    LiteralIndexSegmentSearcher(PrimaryKeyMap.Factory primaryKeyMapFactory,
+                                PerColumnIndexFiles perIndexFiles,
+                                SegmentMetadata segmentMetadata,
+                                IndexContext indexContext) throws IOException
+    {
+        super(primaryKeyMapFactory, perIndexFiles, segmentMetadata, indexContext);
+
+        long root = metadata.getIndexRoot(IndexComponent.TERMS_DATA);
+        assert root >= 0;
+
+        perColumnEventListener = (QueryEventListener.TrieIndexEventListener)indexContext.getColumnQueryMetrics();
+
+        Map<String,String> map = metadata.componentMetadatas.get(IndexComponent.TERMS_DATA).attributes;
+        String footerPointerString = map.get(SAICodecUtils.FOOTER_POINTER);
+        long footerPointer = footerPointerString == null ? -1 : Long.parseLong(footerPointerString);
+
+        reader = new LiteralIndexSegmentTermsReader(indexContext,
+                                                    indexFiles.termsData(),
+                                                    indexFiles.postingLists(),
+                                                    root, footerPointer);
+    }
+
+    @Override
+    public long indexFileCacheSize()
+    {
+        // trie has no pre-allocated memory.
+        return 0;
+    }
+
+    @Override
+    @SuppressWarnings({"resource", "RedundantSuppression"})
+    public KeyRangeIterator search(Expression expression, QueryContext queryContext) throws IOException
+    {
+        if (logger.isTraceEnabled())
+            logger.trace(indexContext.logMessage("Searching on expression '{}'..."), expression);
+
+        if (!expression.getOp().isEquality())
+            throw new IllegalArgumentException(indexContext.logMessage("Unsupported expression: " + expression));
+
+        final ByteComparable term = ByteComparable.fixedLength(expression.lower.value.encoded);
+        QueryEventListener.TrieIndexEventListener listener = MulticastQueryEventListeners.of(queryContext, perColumnEventListener);
+        PostingList postingList = reader.exactMatch(term, listener, queryContext);
+        return toIterator(postingList, queryContext);
+    }
+
+    @Override
+    public String toString()
+    {
+        return MoreObjects.toStringHelper(this)
+                          .add("indexContext", indexContext)
+                          .toString();
+    }
+
+    @Override
+    public void close()
+    {
+        reader.close();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentTermsReader.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentTermsReader.java
new file mode 100644
index 0000000000..abca7c95cb
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/LiteralIndexSegmentTermsReader.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.segment;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.exceptions.QueryCancelledException;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.disk.io.IndexFileUtils;
+import org.apache.cassandra.index.sai.disk.v1.postings.PostingsReader;
+import org.apache.cassandra.index.sai.disk.v1.trie.TrieTermsDictionaryReader;
+import org.apache.cassandra.index.sai.metrics.QueryEventListener;
+import org.apache.cassandra.index.sai.postings.PostingList;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.Throwables;
+import org.apache.cassandra.utils.bytecomparable.ByteComparable;
+import org.apache.lucene.store.IndexInput;
+
+import static org.apache.cassandra.index.sai.disk.v1.SAICodecUtils.validate;
+
+/**
+ * Synchronous reader of terms dictionary and postings lists to produce a {@link PostingList} with matching row ids.
+ *
+ * {@link #exactMatch(ByteComparable, QueryEventListener.TrieIndexEventListener, QueryContext)} does:
+ * <ul>
+ * <li>{@link TermQuery#lookupPostingsOffset(ByteComparable)}: does term dictionary lookup to find the posting list file
+ * position</li>
+ * <li>{@link TermQuery#getPostingsReader(long)}: reads posting list block summary and initializes posting read which
+ * reads the first block of the posting list into memory</li>
+ * </ul>
+ */
+public class LiteralIndexSegmentTermsReader implements Closeable
+{
+    private static final Logger logger = LoggerFactory.getLogger(LiteralIndexSegmentTermsReader.class);
+
+    private final IndexContext indexContext;
+    private final FileHandle termDictionaryFile;
+    private final FileHandle postingsFile;
+    private final long termDictionaryRoot;
+
+    public LiteralIndexSegmentTermsReader(IndexContext indexContext,
+                                          FileHandle termsData,
+                                          FileHandle postingLists,
+                                          long root,
+                                          long termsFooterPointer) throws IOException
+    {
+        this.indexContext = indexContext;
+        termDictionaryFile = termsData;
+        postingsFile = postingLists;
+        termDictionaryRoot = root;
+
+        try (final IndexInput indexInput = IndexFileUtils.instance.openInput(termDictionaryFile))
+        {
+            validate(indexInput, termsFooterPointer);
+        }
+
+        try (final IndexInput indexInput = IndexFileUtils.instance.openInput(postingsFile))
+        {
+            validate(indexInput);
+        }
+    }
+
+    @Override
+    public void close()
+    {
+        FileUtils.closeQuietly(termDictionaryFile);
+        FileUtils.closeQuietly(postingsFile);
+    }
+
+    public PostingList exactMatch(ByteComparable term, QueryEventListener.TrieIndexEventListener perQueryEventListener, QueryContext context)
+    {
+        perQueryEventListener.onSegmentHit();
+        return new TermQuery(term, perQueryEventListener, context).execute();
+    }
+
+    @VisibleForTesting
+    public class TermQuery
+    {
+        private final IndexInput postingsInput;
+        private final IndexInput postingsSummaryInput;
+        private final QueryEventListener.TrieIndexEventListener listener;
+        private final long lookupStartTime;
+        private final QueryContext context;
+        private final ByteComparable term;
+
+        TermQuery(ByteComparable term, QueryEventListener.TrieIndexEventListener listener, QueryContext context)
+        {
+            this.listener = listener;
+            postingsInput = IndexFileUtils.instance.openInput(postingsFile);
+            postingsSummaryInput = IndexFileUtils.instance.openInput(postingsFile);
+            this.term = term;
+            lookupStartTime = Clock.Global.nanoTime();
+            this.context = context;
+        }
+
+        public PostingList execute()
+        {
+            try
+            {
+                long postingOffset = lookupPostingsOffset(term);
+                if (postingOffset == PostingList.OFFSET_NOT_FOUND)
+                {
+                    FileUtils.closeQuietly(postingsInput);
+                    FileUtils.closeQuietly(postingsSummaryInput);
+                    return null;
+                }
+
+                context.checkpoint();
+
+                // when posting is found, resources will be closed when posting reader is closed.
+                return getPostingsReader(postingOffset);
+            }
+            catch (Throwable e)
+            {
+                if (!(e instanceof QueryCancelledException))
+                    logger.error(indexContext.logMessage("Failed to execute term query"), e);
+
+                closeOnException();
+                throw Throwables.cleaned(e);
+            }
+        }
+
+        private void closeOnException()
+        {
+            FileUtils.closeQuietly(postingsInput);
+            FileUtils.closeQuietly(postingsSummaryInput);
+        }
+
+        public long lookupPostingsOffset(ByteComparable term)
+        {
+            try (TrieTermsDictionaryReader reader = new TrieTermsDictionaryReader(termDictionaryFile.instantiateRebufferer(null), termDictionaryRoot))
+            {
+                final long offset = reader.exactMatch(term);
+
+                listener.onTraversalComplete(Clock.Global.nanoTime() - lookupStartTime, TimeUnit.NANOSECONDS);
+
+                if (offset == TrieTermsDictionaryReader.NOT_FOUND)
+                    return PostingList.OFFSET_NOT_FOUND;
+
+                return offset;
+            }
+        }
+
+        public PostingsReader getPostingsReader(long offset) throws IOException
+        {
+            PostingsReader.BlocksSummary header = new PostingsReader.BlocksSummary(postingsSummaryInput, offset);
+
+            return new PostingsReader(postingsInput, header, listener.postingListEventListener());
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/Segment.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/Segment.java
new file mode 100644
index 0000000000..a36a3504dc
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/Segment.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.segment;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.QueryContext;
+import org.apache.cassandra.index.sai.SSTableContext;
+import org.apache.cassandra.index.sai.disk.PrimaryKeyMap;
+import org.apache.cassandra.index.sai.disk.v1.PerColumnIndexFiles;
+import org.apache.cassandra.index.sai.plan.Expression;
+import org.apache.cassandra.index.sai.iterators.KeyRangeIterator;
+import org.apache.cassandra.io.util.FileUtils;
+
+/**
+ * Each segment represents an on-disk index structure (kdtree/terms/postings) flushed by memory limit or token boundaries.
+ * It also helps to reduce resource consumption for read requests as only segments that intersect with read request data
+ * range need to be loaded.
+ */
+public class Segment implements Closeable
+{
+    private final Token minKey;
+    private final Token.KeyBound minKeyBound;
+    private final Token maxKey;
+    private final Token.KeyBound maxKeyBound;
+
+    // per sstable
+    final PrimaryKeyMap.Factory primaryKeyMapFactory;
+    // per-segment
+    public final SegmentMetadata metadata;
+
+    private final IndexSegmentSearcher index;
+
+    public Segment(IndexContext indexContext, SSTableContext sstableContext, PerColumnIndexFiles indexFiles, SegmentMetadata metadata) throws IOException
+    {
+        this.minKey = metadata.minKey.token();
+        this.minKeyBound = minKey.minKeyBound();
+        this.maxKey = metadata.maxKey.token();
+        this.maxKeyBound = maxKey.maxKeyBound();
+
+        this.primaryKeyMapFactory = sstableContext.primaryKeyMapFactory;
+        this.metadata = metadata;
+
+        this.index = IndexSegmentSearcher.open(primaryKeyMapFactory, indexFiles, metadata, indexContext);
+    }
+
+    @VisibleForTesting
+    public Segment(Token minKey, Token maxKey)
+    {
+        this.primaryKeyMapFactory = null;
+        this.metadata = null;
+        this.minKey = minKey;
+        this.minKeyBound = minKey.minKeyBound();
+        this.maxKey = maxKey;
+        this.maxKeyBound = maxKey.maxKeyBound();
+        this.index = null;
+    }
+
+    /**
+     * @return true if current segment intersects with query key range
+     */
+    public boolean intersects(AbstractBounds<PartitionPosition> keyRange)
+    {
+        if (keyRange instanceof Range && ((Range<?>)keyRange).isWrapAround())
+            return keyRange.contains(minKeyBound) || keyRange.contains(maxKeyBound);
+
+        int cmp = keyRange.right.getToken().compareTo(minKey);
+        // if right is minimum, it means right is the max token and bigger than maxKey.
+        // if right bound is less than minKey, no intersection
+        if (!keyRange.right.isMinimum() && (!keyRange.inclusiveRight() && cmp == 0 || cmp < 0))
+            return false;
+
+        // if left bound is bigger than maxKey, no intersection
+        return keyRange.isStartInclusive() || keyRange.left.getToken().compareTo(maxKey) < 0;
+    }
+
+    public long indexFileCacheSize()
+    {
+        return index == null ? 0 : index.indexFileCacheSize();
+    }
+
+    /**
+     * Search on-disk index synchronously
+     *
+     * @param expression to filter on disk index
+     * @param context to track per sstable cache and per query metrics
+
+     * @return range iterator that matches given expression
+     */
+    public KeyRangeIterator search(Expression expression, QueryContext context) throws IOException
+    {
+        return index.search(expression, context);
+    }
+
+    @Override
+    public void close()
+    {
+        FileUtils.closeQuietly(index);
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("Segment{metadata=%s}", metadata);
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentBuilder.java b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentBuilder.java
new file mode 100644
index 0000000000..dd8666213d
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/segment/SegmentBuilder.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sai.disk.v1.segment;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.annotation.concurrent.NotThreadSafe;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.index.sai.IndexContext;
+import org.apache.cassandra.index.sai.memory.RAMStringIndexer;
+import org.apache.cassandra.index.sai.disk.format.IndexDescriptor;
+import org.apache.cassandra.index.sai.disk.v1.trie.LiteralIndexWriter;
+import org.apache.cassandra.index.sai.utils.NamedMemoryLimiter;
+import org.apache.cassandra.index.sai.utils.PrimaryKey;
+import org.apache.cassandra.index.sai.utils.TypeUtil;
+import org.apache.cassandra.utils.FastByteOperations;
+import org.apache.lucene.util.BytesRefBuilder;
+
+/**
+ * Creates an on-heap index data structure to be flushed to an SSTable index.
+ */
+@NotThreadSafe
+public abstract class SegmentBuilder
+{
+    private static final Logger logger = LoggerFactory.getLogger(SegmentBuilder.class);
+
+    // Served as safe net in case memory limit is not triggered or when merger merges small segments..
+    public static final long LAST_VALID_SEGMENT_ROW_ID = (Integer.MAX_VALUE / 2) - 1L;
+    private static long testLastValidSegmentRowId = -1;
+
+    /** The number of column indexes being built globally. (Starts at one to avoid divide by zero.) */
+    private static final AtomicInteger ACTIVE_BUILDER_COUNT = new AtomicInteger(0);
+
+    /** Minimum flush size, dynamically updated as segment builds are started and completed/aborted. */
+    private static volatile long minimumFlushBytes;
+    private final AbstractType<?> termComparator;
+    private final NamedMemoryLimiter limiter;
+    private final long lastValidSegmentRowID;
+    private boolean flushed = false;
+    private boolean active = true;
+    // segment metadata
+    private long minSSTableRowId = -1;
+    private long maxSSTableRowId = -1;
+    private long segmentRowIdOffset = 0;
+
+    // in token order
+    private PrimaryKey minKey;
+    private PrimaryKey maxKey;
+    // in termComparator order
+    private ByteBuffer minTerm;
+    private ByteBuffer maxTerm;
+
+    long totalBytesAllocated;
+    int rowCount = 0;
+    int maxSegmentRowId = -1;
+
+    public static class RAMStringSegmentBuilder extends SegmentBuilder
+    {
+        final RAMStringIndexer ramIndexer;
+
+        final BytesRefBuilder stringBuffer = new BytesRefBuilder();
+
+        public RAMStringSegmentBuilder(AbstractType<?> termComparator, NamedMemoryLimiter limiter)
+        {
+            super(termComparator, limiter);
+
+            ramIndexer = new RAMStringIndexer();
+            totalBytesAllocated = ramIndexer.estimatedBytesUsed();
+        }
+
+        public boolean isEmpty()
+        {
+            return ramIndexer.isEmpty();
+        }
+
+        protected long addInternal(ByteBuffer term, int segmentRowId)
+        {
+            copyBufferToBytesRef(term, stringBuffer);
+            return ramIndexer.add(stringBuffer.get(), segmentRowId);
+        }
+
+        @Override
+        protected SegmentMetadata.ComponentMetadataMap flushInternal(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException
+        {
+            try (LiteralIndexWriter writer = new LiteralIndexWriter(indexDescriptor, indexContext))
+            {
+                return writer.writeCompleteSegment(ramIndexer.getTermsWithPostings());
+            }
+        }
+
+        private void copyBufferToBytesRef(ByteBuffer buffer, BytesRefBuilder stringBuffer)
+        {
+            int length = buffer.remaining();
+            stringBuffer.clear();
+            stringBuffer.grow(length);
+            FastByteOperations.copy(buffer, buffer.position(), stringBuffer.bytes(), 0, length);
+            stringBuffer.setLength(length);
+        }
+    }
+
+    public static int getActiveBuilderCount()
+    {
+        return ACTIVE_BUILDER_COUNT.get();
+    }
+
+    private SegmentBuilder(AbstractType<?> termComparator, NamedMemoryLimiter limiter)
+    {
+        this.termComparator = termComparator;
+        this.limiter = limiter;
+        this.lastValidSegmentRowID = testLastValidSegmentRowId >= 0 ? testLastValidSegmentRowId : LAST_VALID_SEGMENT_ROW_ID;
+
+        minimumFlushBytes = limiter.limitBytes() / ACTIVE_BUILDER_COUNT.incrementAndGet();
+    }
+
+    public SegmentMetadata flush(IndexDescriptor indexDescriptor, IndexContext indexContext) throws IOException
+    {
+        assert !flushed : "Cannot flush an already flushed segment";
+        flushed = true;
+
+        if (getRowCount() == 0)
+        {
+            logger.warn(indexContext.logMessage("No rows to index during flush of SSTable {}."), indexDescriptor.sstableDescriptor);
+            return null;
+        }
+
+        SegmentMetadata.ComponentMetadataMap indexMetas = flushInternal(indexDescriptor, indexContext);
+
+        return new SegmentMetadata(segmentRowIdOffset, rowCount, minSSTableRowId, maxSSTableRowId, minKey, maxKey, minTerm, maxTerm, indexMetas);
+    }
+
+    public long add(ByteBuffer term, PrimaryKey key, long sstableRowId)
+    {
+        assert !flushed : "Cannot add to a flushed segment.";
+        assert sstableRowId >= maxSSTableRowId;
+        minSSTableRowId = minSSTableRowId < 0 ? sstableRowId : minSSTableRowId;
+        maxSSTableRowId = sstableRowId;
+
+        assert maxKey == null || maxKey.compareTo(key) <= 0;
+        if (minKey == null)
+            minKey = key;
+        maxKey = key;
+
+        minTerm = TypeUtil.min(term, minTerm, termComparator);
+        maxTerm = TypeUtil.max(term, maxTerm, termComparator);
+
+        if (rowCount == 0)
+        {
+            // use first global rowId in the segment as segment rowId offset
+            segmentRowIdOffset = sstableRowId;
+        }
... 18337 lines suppressed ...


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


[cassandra] 01/06: Add Index Group Interface for Storage Attached Index

Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

maedhroz pushed a commit to branch cep-7-sai
in repository https://gitbox.apache.org/repos/asf/cassandra.git

commit 07ad4d8491d51517282be12543f1bb43b7d8db71
Author: Zhao Yang <zh...@gmail.com>
AuthorDate: Tue May 19 17:08:36 2020 +0800

    Add Index Group Interface for Storage Attached Index
    
    patch by Zhao Yang; reviewed by Caleb Rackliffe and Andres de la Peña for CASSANDRA-16092
    
    Co-authored-by: Caleb Rackliffe <ca...@gmail.com>
    Co-authored-by: Andres de la Peña <a....@gmail.com>
---
 .../cassandra/db/CassandraTableWriteHandler.java   |   5 +-
 .../org/apache/cassandra/db/ColumnFamilyStore.java |  21 +-
 src/java/org/apache/cassandra/db/Keyspace.java     |   8 +-
 .../cassandra/db/PartitionRangeReadCommand.java    |  38 +-
 src/java/org/apache/cassandra/db/ReadCommand.java  |  94 ++--
 .../cassandra/db/ReadExecutionController.java      |  12 +-
 .../cassandra/db/SinglePartitionReadCommand.java   |  36 +-
 .../org/apache/cassandra/db/TableWriteHandler.java |   3 +-
 .../db/compaction/AbstractCompactionStrategy.java  |  13 +-
 .../db/compaction/AbstractStrategyHolder.java      |   2 +-
 .../db/compaction/CompactionIterator.java          |   5 +-
 .../cassandra/db/compaction/CompactionManager.java |   8 +-
 .../db/compaction/CompactionStrategyHolder.java    |   4 +-
 .../db/compaction/CompactionStrategyManager.java   |   4 +-
 .../db/compaction/PendingRepairHolder.java         |   4 +-
 .../apache/cassandra/db/compaction/Upgrader.java   |  10 +-
 .../compaction/writers/CompactionAwareWriter.java  |   9 +-
 .../writers/DefaultCompactionWriter.java           |  10 +-
 .../writers/MajorLeveledCompactionWriter.java      |   6 +-
 .../compaction/writers/MaxSSTableSizeWriter.java   |   2 +-
 .../SplittingSizeTieredCompactionWriter.java       |   2 +-
 src/java/org/apache/cassandra/index/Index.java     | 380 ++++++++++++--
 .../org/apache/cassandra/index/IndexRegistry.java  |  97 +++-
 .../cassandra/index/SecondaryIndexManager.java     | 306 ++++++++---
 .../cassandra/index/SingletonIndexGroup.java       | 114 +++++
 .../cassandra/index/SingletonIndexQueryPlan.java   |  87 ++++
 .../cassandra/index/internal/CassandraIndex.java   |  25 +-
 .../index/internal/CassandraIndexSearcher.java     |   6 +
 .../org/apache/cassandra/index/sasi/SASIIndex.java |  30 +-
 .../sasi/SASIIndexGroup.java}                      |  21 +-
 .../index/sasi/disk/PerSSTableIndexWriter.java     |  10 +-
 .../index/sasi/plan/SASIIndexQueryPlan.java        |  51 ++
 .../{QueryPlan.java => SASIIndexSearcher.java}     |  17 +-
 .../org/apache/cassandra/io/sstable/SSTable.java   |   2 -
 .../cassandra/io/sstable/SSTableFlushObserver.java |  21 +-
 .../cassandra/io/sstable/SSTableTxnWriter.java     |  10 +-
 .../io/sstable/SimpleSSTableMultiWriter.java       |   8 +-
 .../cassandra/io/sstable/format/SSTableWriter.java |  49 +-
 .../io/sstable/format/SortedTableWriter.java       |   6 +-
 .../cassandra/io/sstable/format/big/BigFormat.java |   2 +-
 .../io/sstable/format/big/BigTableWriter.java      |  10 +-
 .../metrics/ClientRangeRequestMetrics.java         |  47 ++
 .../org/apache/cassandra/schema/IndexMetadata.java |  22 +-
 .../paxos/uncommitted/PaxosUncommittedIndex.java   |   2 +-
 .../cassandra/service/reads/DataResolver.java      |  32 +-
 .../service/reads/ShortReadRowsProtection.java     |   2 +-
 .../service/reads/range/RangeCommandIterator.java  |   5 +-
 .../service/reads/range/RangeCommands.java         |   2 +-
 .../upgrade/MixedModeIndexTestBase.java            | 135 +++++
 .../distributed/upgrade/MixedModeIndexV40Test.java |  16 +-
 .../apache/cassandra/db/SecondaryIndexTest.java    |  12 +-
 .../cassandra/db/SerializationHeaderTest.java      |   6 +-
 .../db/compaction/ActiveCompactionsTest.java       |   4 +-
 .../db/lifecycle/RealTransactionsTest.java         |   4 +-
 .../apache/cassandra/index/CustomIndexTest.java    | 563 ++++++++++++++++++++-
 .../cassandra/index/SecondaryIndexManagerTest.java |   4 +-
 .../unit/org/apache/cassandra/index/StubIndex.java |  30 +-
 .../org/apache/cassandra/index/StubIndexGroup.java |  99 ++++
 .../index/internal/CustomCassandraIndex.java       |  19 +-
 .../apache/cassandra/index/sasi/SASIIndexTest.java |   6 +-
 .../index/sasi/disk/PerSSTableIndexWriterTest.java |   5 +-
 .../io/sstable/SSTableFlushObserverTest.java       |   5 +-
 .../cassandra/io/sstable/SSTableReaderTest.java    |   2 +-
 .../io/sstable/SSTableWriterTestBase.java          |   4 +-
 .../org/apache/cassandra/io/sstable/ScrubTest.java |   2 +-
 .../metrics/ClientRequestMetricsTest.java          |  23 +-
 66 files changed, 2145 insertions(+), 454 deletions(-)

diff --git a/src/java/org/apache/cassandra/db/CassandraTableWriteHandler.java b/src/java/org/apache/cassandra/db/CassandraTableWriteHandler.java
index 146539c26e..671c06d1e5 100644
--- a/src/java/org/apache/cassandra/db/CassandraTableWriteHandler.java
+++ b/src/java/org/apache/cassandra/db/CassandraTableWriteHandler.java
@@ -19,7 +19,6 @@
 package org.apache.cassandra.db;
 
 import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.index.transactions.UpdateTransaction;
 import org.apache.cassandra.tracing.Tracing;
 
 public class CassandraTableWriteHandler implements TableWriteHandler
@@ -33,10 +32,10 @@ public class CassandraTableWriteHandler implements TableWriteHandler
 
     @Override
     @SuppressWarnings("resource")
-    public void write(PartitionUpdate update, WriteContext context, UpdateTransaction updateTransaction)
+    public void write(PartitionUpdate update, WriteContext context, boolean updateIndexes)
     {
         CassandraWriteContext ctx = CassandraWriteContext.fromContext(context);
         Tracing.trace("Adding to {} memtable", update.metadata().name);
-        cfs.apply(update, updateTransaction, ctx.getGroup(), ctx.getPosition());
+        cfs.apply(update, ctx, updateIndexes);
     }
 }
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 1be2085779..704a7096c8 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -654,7 +654,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
 
     public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, MetadataCollector metadataCollector, SerializationHeader header, LifecycleNewTracker lifecycleNewTracker)
     {
-        return getCompactionStrategyManager().createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadataCollector, header, indexManager.listIndexes(), lifecycleNewTracker);
+        return getCompactionStrategyManager().createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadataCollector, header, indexManager.listIndexGroups(), lifecycleNewTracker);
     }
 
     public boolean supportsEarlyOpen()
@@ -1409,17 +1409,21 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
     /**
      * Insert/Update the column family for this key.
      * Caller is responsible for acquiring Keyspace.switchLock
-     * param @ lock - lock that needs to be used.
-     * param @ key - key for update/insert
-     * param @ columnFamily - columnFamily changes
+     * @param update to be applied
+     * @param context write context for current update
+     * @param updateIndexes whether secondary indexes should be updated
      */
-    public void apply(PartitionUpdate update, UpdateTransaction indexer, OpOrder.Group opGroup, CommitLogPosition commitLogPosition)
+    @SuppressWarnings("resource") // opGroup
+    public void apply(PartitionUpdate update, CassandraWriteContext context, boolean updateIndexes)
 
     {
         long start = nanoTime();
+        OpOrder.Group opGroup = context.getGroup();
+        CommitLogPosition commitLogPosition = context.getPosition();
         try
         {
             Memtable mt = data.getMemtableFor(opGroup, commitLogPosition);
+            UpdateTransaction indexer = newUpdateTransaction(update, context, updateIndexes, mt);
             long timeDelta = mt.put(update, indexer, opGroup);
             DecoratedKey key = update.partitionKey();
             invalidateCachedPartition(key);
@@ -1444,6 +1448,13 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
         }
     }
 
+    private UpdateTransaction newUpdateTransaction(PartitionUpdate update, CassandraWriteContext context, boolean updateIndexes, Memtable memtable)
+    {
+        return updateIndexes
+               ? indexManager.newUpdateTransaction(update, context, FBUtilities.nowInSeconds(), memtable)
+               : UpdateTransaction.NO_OP;
+    }
+
     @Override
     public ShardBoundaries localRangeSplits(int shardCount)
     {
diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java
index 22721bdcf1..a3f05e6048 100644
--- a/src/java/org/apache/cassandra/db/Keyspace.java
+++ b/src/java/org/apache/cassandra/db/Keyspace.java
@@ -50,7 +50,6 @@ import org.apache.cassandra.db.view.ViewManager;
 import org.apache.cassandra.exceptions.WriteTimeoutException;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.SecondaryIndexManager;
-import org.apache.cassandra.index.transactions.UpdateTransaction;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.locator.AbstractReplicationStrategy;
@@ -67,7 +66,6 @@ import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.service.snapshot.TableSnapshot;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.concurrent.AsyncPromise;
 import org.apache.cassandra.utils.concurrent.Future;
@@ -611,7 +609,6 @@ public class Keyspace
                     columnFamilyStores.get(tableId).metric.viewLockAcquireTime.update(acquireTime, MILLISECONDS);
             }
         }
-        int nowInSec = FBUtilities.nowInSeconds();
         try (WriteContext ctx = getWriteHandler().beginWrite(mutation, makeDurable))
         {
             for (PartitionUpdate upd : mutation.getPartitionUpdates())
@@ -640,10 +637,7 @@ public class Keyspace
                     }
                 }
 
-                UpdateTransaction indexTransaction = updateIndexes
-                                                     ? cfs.indexManager.newUpdateTransaction(upd, ctx, nowInSec)
-                                                     : UpdateTransaction.NO_OP;
-                cfs.getWriteHandler().write(upd, ctx, indexTransaction);
+                cfs.getWriteHandler().write(upd, ctx, updateIndexes);
 
                 if (requiresViewUpdate)
                     baseComplete.set(currentTimeMillis());
diff --git a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
index 164e03cf65..4a47a96189 100644
--- a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
+++ b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
@@ -50,7 +50,6 @@ import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.metrics.TableMetrics;
 import org.apache.cassandra.net.Verb;
-import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.ClientState;
 import org.apache.cassandra.service.StorageProxy;
@@ -75,10 +74,10 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                                       RowFilter rowFilter,
                                       DataLimits limits,
                                       DataRange dataRange,
-                                      IndexMetadata index,
+                                      Index.QueryPlan indexQueryPlan,
                                       boolean trackWarnings)
     {
-        super(Kind.PARTITION_RANGE, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, index, trackWarnings);
+        super(Kind.PARTITION_RANGE, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan, trackWarnings);
         this.dataRange = dataRange;
         this.requestedSlices = dataRange.clusteringIndexFilter.getSlices(metadata());
 
@@ -93,7 +92,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                                                     RowFilter rowFilter,
                                                     DataLimits limits,
                                                     DataRange dataRange,
-                                                    IndexMetadata index,
+                                                    Index.QueryPlan indexQueryPlan,
                                                     boolean trackWarnings)
     {
         if (metadata.isVirtual())
@@ -107,7 +106,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                                                              rowFilter,
                                                              limits,
                                                              dataRange,
-                                                             index,
+                                                             indexQueryPlan,
                                                              trackWarnings);
         }
         return new PartitionRangeReadCommand(isDigest,
@@ -119,7 +118,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                                              rowFilter,
                                              limits,
                                              dataRange,
-                                             index,
+                                             indexQueryPlan,
                                              trackWarnings);
     }
 
@@ -139,7 +138,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                       rowFilter,
                       limits,
                       dataRange,
-                      findIndex(metadata, rowFilter),
+                      findIndexQueryPlan(metadata, rowFilter),
                       false);
     }
 
@@ -212,7 +211,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                       rowFilter(),
                       isRangeContinuation ? limits() : limits().withoutState(),
                       dataRange().forSubRange(range),
-                      indexMetadata(),
+                      indexQueryPlan(),
                       isTrackingWarnings());
     }
 
@@ -227,7 +226,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                       rowFilter(),
                       limits(),
                       dataRange(),
-                      indexMetadata(),
+                      indexQueryPlan(),
                       isTrackingWarnings());
     }
 
@@ -243,7 +242,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                       rowFilter(),
                       limits(),
                       dataRange(),
-                      indexMetadata(),
+                      indexQueryPlan(),
                       isTrackingWarnings());
     }
 
@@ -259,7 +258,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                       rowFilter(),
                       limits(),
                       dataRange(),
-                      indexMetadata(),
+                      indexQueryPlan(),
                       isTrackingWarnings());
     }
 
@@ -275,7 +274,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                       rowFilter(),
                       newLimits,
                       dataRange(),
-                      indexMetadata(),
+                      indexQueryPlan(),
                       isTrackingWarnings());
     }
 
@@ -291,7 +290,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                       rowFilter(),
                       newLimits,
                       newDataRange,
-                      indexMetadata(),
+                      indexQueryPlan(),
                       isTrackingWarnings());
     }
 
@@ -472,9 +471,8 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
      */
     public PartitionIterator postReconciliationProcessing(PartitionIterator result)
     {
-        ColumnFamilyStore cfs = Keyspace.open(metadata().keyspace).getColumnFamilyStore(metadata().name);
-        Index index = getIndex(cfs);
-        return index == null ? result : index.postProcessorFor(this).apply(result, this);
+        Index.QueryPlan queryPlan = indexQueryPlan();
+        return queryPlan == null ? result : queryPlan.postProcessor().apply(result);
     }
 
     @Override
@@ -528,11 +526,11 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                                        ColumnFilter columnFilter,
                                        RowFilter rowFilter,
                                        DataLimits limits,
-                                       IndexMetadata index)
+                                       Index.QueryPlan indexQueryPlan)
         throws IOException
         {
             DataRange range = DataRange.serializer.deserialize(in, version, metadata);
-            return PartitionRangeReadCommand.create(isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, range, index, false);
+            return new PartitionRangeReadCommand(isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, range, indexQueryPlan, false);
         }
     }
 
@@ -547,10 +545,10 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
                                                       RowFilter rowFilter,
                                                       DataLimits limits,
                                                       DataRange dataRange,
-                                                      IndexMetadata index,
+                                                      Index.QueryPlan indexQueryPlan,
                                                       boolean trackWarnings)
         {
-            super(isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, dataRange, index, trackWarnings);
+            super(isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, dataRange, indexQueryPlan, trackWarnings);
         }
 
         @Override
diff --git a/src/java/org/apache/cassandra/db/ReadCommand.java b/src/java/org/apache/cassandra/db/ReadCommand.java
index 9cfcb3a3d3..abbf099704 100644
--- a/src/java/org/apache/cassandra/db/ReadCommand.java
+++ b/src/java/org/apache/cassandra/db/ReadCommand.java
@@ -23,6 +23,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.function.BiFunction;
 import java.util.function.LongPredicate;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import javax.annotation.Nullable;
 
@@ -49,8 +50,6 @@ import org.apache.cassandra.db.transform.StoppingTransformation;
 import org.apache.cassandra.db.transform.Transformation;
 import org.apache.cassandra.exceptions.UnknownIndexException;
 import org.apache.cassandra.index.Index;
-import org.apache.cassandra.index.IndexNotAvailableException;
-import org.apache.cassandra.index.IndexRegistry;
 import org.apache.cassandra.io.IVersionedSerializer;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.util.DataInputPlus;
@@ -104,7 +103,7 @@ public abstract class ReadCommand extends AbstractReadQuery
     private boolean trackWarnings;
 
     @Nullable
-    private final IndexMetadata index;
+    private final Index.QueryPlan indexQueryPlan;
 
     protected static abstract class SelectionDeserializer
     {
@@ -118,7 +117,7 @@ public abstract class ReadCommand extends AbstractReadQuery
                                                 ColumnFilter columnFilter,
                                                 RowFilter rowFilter,
                                                 DataLimits limits,
-                                                IndexMetadata index) throws IOException;
+                                                Index.QueryPlan indexQueryPlan) throws IOException;
     }
 
     protected enum Kind
@@ -143,7 +142,7 @@ public abstract class ReadCommand extends AbstractReadQuery
                           ColumnFilter columnFilter,
                           RowFilter rowFilter,
                           DataLimits limits,
-                          IndexMetadata index,
+                          Index.QueryPlan indexQueryPlan,
                           boolean trackWarnings)
     {
         super(metadata, nowInSec, columnFilter, rowFilter, limits);
@@ -154,7 +153,7 @@ public abstract class ReadCommand extends AbstractReadQuery
         this.isDigestQuery = isDigestQuery;
         this.digestVersion = digestVersion;
         this.acceptsTransient = acceptsTransient;
-        this.index = index;
+        this.indexQueryPlan = indexQueryPlan;
         this.trackWarnings = trackWarnings;
     }
 
@@ -242,14 +241,20 @@ public abstract class ReadCommand extends AbstractReadQuery
     }
 
     /**
-     * Index (metadata) chosen for this query. Can be null.
+     * Index query plan chosen for this query. Can be null.
      *
-     * @return index (metadata) chosen for this query
+     * @return index query plan chosen for this query
      */
     @Nullable
-    public IndexMetadata indexMetadata()
+    public Index.QueryPlan indexQueryPlan()
     {
-        return index;
+        return indexQueryPlan;
+    }
+
+    @VisibleForTesting
+    public Index.Searcher indexSearcher()
+    {
+        return indexQueryPlan == null ? null : indexQueryPlan.searcherFor(this);
     }
 
     /**
@@ -350,30 +355,19 @@ public abstract class ReadCommand extends AbstractReadQuery
 
     long indexSerializedSize(int version)
     {
-        return null != index
-             ? IndexMetadata.serializer.serializedSize(index, version)
+        return null != indexQueryPlan
+             ? IndexMetadata.serializer.serializedSize(indexQueryPlan.getFirst().getIndexMetadata(), version)
              : 0;
     }
 
-    public Index getIndex(ColumnFamilyStore cfs)
-    {
-        return null != index
-             ? cfs.indexManager.getIndex(index)
-             : null;
-    }
-
-    static IndexMetadata findIndex(TableMetadata table, RowFilter rowFilter)
+    static Index.QueryPlan findIndexQueryPlan(TableMetadata table, RowFilter rowFilter)
     {
         if (table.indexes.isEmpty() || rowFilter.isEmpty())
             return null;
 
         ColumnFamilyStore cfs = Keyspace.openAndGetStore(table);
 
-        Index index = cfs.indexManager.getBestIndexFor(rowFilter);
-
-        return null != index
-             ? index.getIndexMetadata()
-             : null;
+        return cfs.indexManager.getBestIndexQueryPlanFor(rowFilter);
     }
 
     /**
@@ -384,8 +378,10 @@ public abstract class ReadCommand extends AbstractReadQuery
      */
     public void maybeValidateIndex()
     {
-        if (null != index)
-            IndexRegistry.obtain(metadata()).getIndex(index).validate(this);
+        if (null != indexQueryPlan)
+        {
+            indexQueryPlan.validate(this);
+        }
     }
 
     /**
@@ -405,16 +401,23 @@ public abstract class ReadCommand extends AbstractReadQuery
         try
         {
             ColumnFamilyStore cfs = Keyspace.openAndGetStore(metadata());
-            Index index = getIndex(cfs);
+            Index.QueryPlan indexQueryPlan = indexQueryPlan();
 
             Index.Searcher searcher = null;
-            if (index != null)
+            if (indexQueryPlan != null)
             {
-                if (!cfs.indexManager.isIndexQueryable(index))
-                    throw new IndexNotAvailableException(index);
-
-                searcher = index.searcherFor(this);
-                Tracing.trace("Executing read on {}.{} using index {}", cfs.metadata.keyspace, cfs.metadata.name, index.getIndexMetadata().name);
+                cfs.indexManager.checkQueryability(indexQueryPlan);
+
+                Index index = indexQueryPlan.getFirst();
+                searcher = indexQueryPlan.searcherFor(this);
+                Tracing.trace("Executing read on {}.{} using index{} {}",
+                              cfs.metadata.keyspace,
+                              cfs.metadata.name,
+                              indexQueryPlan.getIndexes().size() == 1 ? "" : "es",
+                              indexQueryPlan.getIndexes()
+                                            .stream()
+                                            .map(i -> i.getIndexMetadata().name)
+                                            .collect(Collectors.joining(",")));
             }
 
             UnfilteredPartitionIterator iterator = (null == searcher) ? queryStorage(cfs, executionController) : searcher.search(executionController);
@@ -430,7 +433,7 @@ public abstract class ReadCommand extends AbstractReadQuery
 
                 // If we've used a 2ndary index, we know the result already satisfy the primary expression used, so
                 // no point in checking it again.
-                RowFilter filter = (null == searcher) ? rowFilter() : index.getPostIndexQueryFilter(rowFilter());
+                RowFilter filter = (null == searcher) ? rowFilter() : indexQueryPlan.postIndexQueryFilter();
 
                 /*
                  * TODO: We'll currently do filtering by the rowFilter here because it's convenient. However,
@@ -1064,7 +1067,7 @@ public abstract class ReadCommand extends AbstractReadQuery
             out.writeByte(command.kind.ordinal());
             out.writeByte(
                     digestFlag(command.isDigestQuery())
-                    | indexFlag(null != command.indexMetadata())
+                    | indexFlag(null != command.indexQueryPlan())
                     | acceptsTransientFlag(command.acceptsTransient())
             );
             if (command.isDigestQuery())
@@ -1074,8 +1077,11 @@ public abstract class ReadCommand extends AbstractReadQuery
             ColumnFilter.serializer.serialize(command.columnFilter(), out, version);
             RowFilter.serializer.serialize(command.rowFilter(), out, version);
             DataLimits.serializer.serialize(command.limits(), out, version, command.metadata().comparator);
-            if (null != command.index)
-                IndexMetadata.serializer.serialize(command.index, out, version);
+            // Using the name of one of the indexes in the plan to identify the index group because we want
+            // to keep compatibility with legacy nodes. Each replica can create its own different index query plan
+            // from the index name.
+            if (null != command.indexQueryPlan)
+                IndexMetadata.serializer.serialize(command.indexQueryPlan.getFirst().getIndexMetadata(), out, version);
 
             command.serializeSelection(out, version);
         }
@@ -1101,9 +1107,17 @@ public abstract class ReadCommand extends AbstractReadQuery
             ColumnFilter columnFilter = ColumnFilter.serializer.deserialize(in, version, metadata);
             RowFilter rowFilter = RowFilter.serializer.deserialize(in, version, metadata);
             DataLimits limits = DataLimits.serializer.deserialize(in, version,  metadata);
-            IndexMetadata index = hasIndex ? deserializeIndexMetadata(in, version, metadata) : null;
 
-            return kind.selectionDeserializer.deserialize(in, version, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, index);
+            Index.QueryPlan indexQueryPlan = null;
+            if (hasIndex)
+            {
+                IndexMetadata index = deserializeIndexMetadata(in, version, metadata);
+                Index.Group indexGroup =  Keyspace.openAndGetStore(metadata).indexManager.getIndexGroup(index);
+                if (indexGroup != null)
+                    indexQueryPlan = indexGroup.queryPlanFor(rowFilter);
+            }
+
+            return kind.selectionDeserializer.deserialize(in, version, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan);
         }
 
         private IndexMetadata deserializeIndexMetadata(DataInputPlus in, int version, TableMetadata metadata) throws IOException
diff --git a/src/java/org/apache/cassandra/db/ReadExecutionController.java b/src/java/org/apache/cassandra/db/ReadExecutionController.java
index 2fbe3ac610..94740f0a20 100644
--- a/src/java/org/apache/cassandra/db/ReadExecutionController.java
+++ b/src/java/org/apache/cassandra/db/ReadExecutionController.java
@@ -124,7 +124,7 @@ public class ReadExecutionController implements AutoCloseable
     static ReadExecutionController forCommand(ReadCommand command, boolean trackRepairedStatus)
     {
         ColumnFamilyStore baseCfs = Keyspace.openAndGetStore(command.metadata());
-        ColumnFamilyStore indexCfs = maybeGetIndexCfs(baseCfs, command);
+        ColumnFamilyStore indexCfs = maybeGetIndexCfs(command);
 
         long createdAtNanos = baseCfs.metric.topLocalReadQueryTime.isEnabled() ? clock.now() : NO_SAMPLING;
 
@@ -165,10 +165,14 @@ public class ReadExecutionController implements AutoCloseable
         }
     }
 
-    private static ColumnFamilyStore maybeGetIndexCfs(ColumnFamilyStore baseCfs, ReadCommand command)
+    private static ColumnFamilyStore maybeGetIndexCfs(ReadCommand command)
     {
-        Index index = command.getIndex(baseCfs);
-        return index == null ? null : index.getBackingTable().orElse(null);
+        Index.QueryPlan queryPlan = command.indexQueryPlan();
+        if (queryPlan == null)
+            return null;
+
+        // only the index groups with a single member are allowed to have a backing table
+        return queryPlan.getFirst().getBackingTable().orElse(null);
     }
 
     public TableMetadata metadata()
diff --git a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
index d85e2fe628..ba97e12b5b 100644
--- a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
+++ b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
@@ -64,6 +64,7 @@ import org.apache.cassandra.db.transform.Transformation;
 import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
 import org.apache.cassandra.db.virtual.VirtualTable;
 import org.apache.cassandra.exceptions.RequestExecutionException;
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.io.sstable.SSTableReadsListener;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.util.DataInputPlus;
@@ -71,7 +72,6 @@ import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.metrics.TableMetrics;
 import org.apache.cassandra.net.Verb;
 import org.apache.cassandra.schema.ColumnMetadata;
-import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.CacheService;
 import org.apache.cassandra.service.ClientState;
@@ -101,10 +101,10 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                          DataLimits limits,
                                          DecoratedKey partitionKey,
                                          ClusteringIndexFilter clusteringIndexFilter,
-                                         IndexMetadata index,
+                                         Index.QueryPlan indexQueryPlan,
                                          boolean trackWarnings)
     {
-        super(Kind.SINGLE_PARTITION, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, index, trackWarnings);
+        super(Kind.SINGLE_PARTITION, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan, trackWarnings);
         assert partitionKey.getPartitioner() == metadata.partitioner;
         this.partitionKey = partitionKey;
         this.clusteringIndexFilter = clusteringIndexFilter;
@@ -120,7 +120,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                                     DataLimits limits,
                                                     DecoratedKey partitionKey,
                                                     ClusteringIndexFilter clusteringIndexFilter,
-                                                    IndexMetadata index,
+                                                    Index.QueryPlan indexQueryPlan,
                                                     boolean trackWarnings)
     {
         if (metadata.isVirtual())
@@ -135,7 +135,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                                               limits,
                                                               partitionKey,
                                                               clusteringIndexFilter,
-                                                              index,
+                                                              indexQueryPlan,
                                                               trackWarnings);
         }
         return new SinglePartitionReadCommand(isDigest,
@@ -148,7 +148,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                               limits,
                                               partitionKey,
                                               clusteringIndexFilter,
-                                              index,
+                                              indexQueryPlan,
                                               trackWarnings);
     }
 
@@ -162,7 +162,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
      * @param limits the limits to use for the query.
      * @param partitionKey the partition key for the partition to query.
      * @param clusteringIndexFilter the clustering index filter to use for the query.
-     * @param indexMetadata explicitly specified index to use for the query
+     * @param indexQueryPlan explicitly specified index to use for the query
      *
      * @return a newly created read command.
      */
@@ -173,7 +173,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                                     DataLimits limits,
                                                     DecoratedKey partitionKey,
                                                     ClusteringIndexFilter clusteringIndexFilter,
-                                                    IndexMetadata indexMetadata)
+                                                    Index.QueryPlan indexQueryPlan)
     {
         return create(false,
                       0,
@@ -185,7 +185,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                       limits,
                       partitionKey,
                       clusteringIndexFilter,
-                      indexMetadata,
+                      indexQueryPlan,
                       false);
     }
 
@@ -217,7 +217,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                       limits,
                       partitionKey,
                       clusteringIndexFilter,
-                      findIndex(metadata, rowFilter));
+                      findIndexQueryPlan(metadata, rowFilter));
     }
 
     /**
@@ -362,7 +362,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                       limits(),
                       partitionKey(),
                       clusteringIndexFilter(),
-                      indexMetadata(),
+                      indexQueryPlan(),
                       isTrackingWarnings());
     }
 
@@ -379,7 +379,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                       limits(),
                       partitionKey(),
                       clusteringIndexFilter(),
-                      indexMetadata(),
+                      indexQueryPlan(),
                       isTrackingWarnings());
     }
 
@@ -396,7 +396,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                       limits(),
                       partitionKey(),
                       clusteringIndexFilter(),
-                      indexMetadata(),
+                      indexQueryPlan(),
                       isTrackingWarnings());
     }
 
@@ -413,7 +413,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                       newLimits,
                       partitionKey(),
                       clusteringIndexFilter(),
-                      indexMetadata(),
+                      indexQueryPlan(),
                       isTrackingWarnings());
     }
 
@@ -1318,12 +1318,12 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                        ColumnFilter columnFilter,
                                        RowFilter rowFilter,
                                        DataLimits limits,
-                                       IndexMetadata index)
+                                       Index.QueryPlan indexQueryPlan)
         throws IOException
         {
             DecoratedKey key = metadata.partitioner.decorateKey(metadata.partitionKeyType.readBuffer(in, DatabaseDescriptor.getMaxValueSize()));
             ClusteringIndexFilter filter = ClusteringIndexFilter.serializer.deserialize(in, version, metadata);
-            return SinglePartitionReadCommand.create(isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, key, filter, index, false);
+            return SinglePartitionReadCommand.create(isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, key, filter, indexQueryPlan, false);
         }
     }
 
@@ -1367,10 +1367,10 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
                                                          DataLimits limits,
                                                          DecoratedKey partitionKey,
                                                          ClusteringIndexFilter clusteringIndexFilter,
-                                                         IndexMetadata index,
+                                                         Index.QueryPlan indexQueryPlan,
                                                          boolean trackWarnings)
         {
-            super(isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, partitionKey, clusteringIndexFilter, index, trackWarnings);
+            super(isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, partitionKey, clusteringIndexFilter, indexQueryPlan, trackWarnings);
         }
 
         @Override
diff --git a/src/java/org/apache/cassandra/db/TableWriteHandler.java b/src/java/org/apache/cassandra/db/TableWriteHandler.java
index 4e47221925..7aa709fe52 100644
--- a/src/java/org/apache/cassandra/db/TableWriteHandler.java
+++ b/src/java/org/apache/cassandra/db/TableWriteHandler.java
@@ -19,9 +19,8 @@
 package org.apache.cassandra.db;
 
 import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.index.transactions.UpdateTransaction;
 
 public interface TableWriteHandler
 {
-    void write(PartitionUpdate update, WriteContext context, UpdateTransaction updateTransaction);
+    void write(PartitionUpdate update, WriteContext context, boolean updateIndexes);
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
index 2eb119dad3..1bcfd4cc94 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
@@ -252,10 +252,10 @@ public abstract class AbstractCompactionStrategy
      * allow for a more memory efficient solution if we know the sstable don't overlap (see
      * LeveledCompactionStrategy for instance).
      */
-    @SuppressWarnings("resource")
+    @SuppressWarnings({"resource", "RedundantSuppression"})
     public ScannerList getScanners(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges)
     {
-        ArrayList<ISSTableScanner> scanners = new ArrayList<ISSTableScanner>();
+        ArrayList<ISSTableScanner> scanners = new ArrayList<>();
         try
         {
             for (SSTableReader sstable : sstables)
@@ -325,9 +325,6 @@ public abstract class AbstractCompactionStrategy
      *
      * Not called when repair status changes (which is also metadata), because this results in the
      * sstable getting removed from the compaction strategy instance.
-     *
-     * @param oldMetadata
-     * @param sstable
      */
     public void metadataChanged(StatsMetadata oldMetadata, SSTableReader sstable)
     {
@@ -366,7 +363,7 @@ public abstract class AbstractCompactionStrategy
 
             for (int i=0, isize=scanners.size(); i<isize; i++)
             {
-                @SuppressWarnings("resource")
+                @SuppressWarnings({"resource", "RedundantSuppression"})
                 ISSTableScanner scanner = scanners.get(i);
                 compressed += scanner.getCompressedLengthInBytes();
                 uncompressed += scanner.getLengthInBytes();
@@ -562,10 +559,10 @@ public abstract class AbstractCompactionStrategy
                                                        boolean isTransient,
                                                        MetadataCollector meta,
                                                        SerializationHeader header,
-                                                       Collection<Index> indexes,
+                                                       Collection<Index.Group> indexGroups,
                                                        LifecycleNewTracker lifecycleNewTracker)
     {
-        return SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, cfs.metadata, meta, header, indexes, lifecycleNewTracker, cfs);
+        return SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, cfs.metadata, meta, header, indexGroups, lifecycleNewTracker, cfs);
     }
 
     public boolean supportsEarlyOpen()
diff --git a/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java b/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java
index 3421123935..600233fabd 100644
--- a/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java
+++ b/src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java
@@ -196,7 +196,7 @@ public abstract class AbstractStrategyHolder
                                                                 boolean isTransient,
                                                                 MetadataCollector collector,
                                                                 SerializationHeader header,
-                                                                Collection<Index> indexes,
+                                                                Collection<Index.Group> indexGroups,
                                                                 LifecycleNewTracker lifecycleNewTracker);
 
     /**
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java b/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java
index 5cde41e1bc..ab03a4f79b 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionIterator.java
@@ -38,6 +38,7 @@ import org.apache.cassandra.db.EmptyIterators;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.RegularAndStaticColumns;
 import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.transform.DuplicateRowChecker;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.partitions.PurgeFunction;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
@@ -50,10 +51,10 @@ import org.apache.cassandra.db.rows.Unfiltered;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.db.rows.UnfilteredRowIterators;
 import org.apache.cassandra.db.rows.WrappingUnfilteredRowIterator;
-import org.apache.cassandra.db.transform.DuplicateRowChecker;
 import org.apache.cassandra.db.transform.Transformation;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.index.transactions.CompactionTransaction;
+import org.apache.cassandra.index.transactions.IndexTransaction;
 import org.apache.cassandra.io.sstable.ISSTableScanner;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.metrics.TopPartitionTracker;
@@ -221,7 +222,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
                 CompactionIterator.this.updateCounterFor(merged);
 
                 if ( (type != OperationType.COMPACTION && type != OperationType.MAJOR_COMPACTION) 
-                    || !controller.cfs.indexManager.hasIndexes() ) 
+                    || !controller.cfs.indexManager.handles(IndexTransaction.Type.COMPACTION) ) 
                 {
                     return null;
                 }
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
index a9724153f6..136497f641 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
@@ -1600,8 +1600,8 @@ public class CompactionManager implements CompactionManagerMBean
                          .setTableMetadataRef(cfs.metadata)
                          .setMetadataCollector(new MetadataCollector(cfs.metadata().comparator).sstableLevel(sstable.getSSTableLevel()))
                          .setSerializationHeader(sstable.header)
-                         .addDefaultComponents()
-                         .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexes(), txn.opType())
+                         .addDefaultComponents(cfs.indexManager.listIndexGroups())
+                         .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexGroups(), txn, cfs.metadata.get())
                          .build(txn, cfs);
     }
 
@@ -1640,8 +1640,8 @@ public class CompactionManager implements CompactionManagerMBean
                          .setTableMetadataRef(cfs.metadata)
                          .setMetadataCollector(new MetadataCollector(sstables, cfs.metadata().comparator, minLevel))
                          .setSerializationHeader(SerializationHeader.make(cfs.metadata(), sstables))
-                         .addDefaultComponents()
-                         .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexes(), txn.opType())
+                         .addDefaultComponents(cfs.indexManager.listIndexGroups())
+                         .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexGroups(), txn, cfs.metadata.get())
                          .build(txn, cfs);
     }
 
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java
index 29e8c296d2..3915a646bd 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java
@@ -222,7 +222,7 @@ public class CompactionStrategyHolder extends AbstractStrategyHolder
                                                        boolean isTransient,
                                                        MetadataCollector collector,
                                                        SerializationHeader header,
-                                                       Collection<Index> indexes,
+                                                       Collection<Index.Group> indexGroups,
                                                        LifecycleNewTracker lifecycleNewTracker)
     {
         if (isRepaired)
@@ -246,7 +246,7 @@ public class CompactionStrategyHolder extends AbstractStrategyHolder
                                                  isTransient,
                                                  collector,
                                                  header,
-                                                 indexes,
+                                                 indexGroups,
                                                  lifecycleNewTracker);
     }
 
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
index ea312bc025..df82341f64 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
@@ -1235,7 +1235,7 @@ public class CompactionStrategyManager implements INotificationConsumer
                                                        boolean isTransient,
                                                        MetadataCollector collector,
                                                        SerializationHeader header,
-                                                       Collection<Index> indexes,
+                                                       Collection<Index.Group> indexGroups,
                                                        LifecycleNewTracker lifecycleNewTracker)
     {
         SSTable.validateRepairedMetadata(repairedAt, pendingRepair, isTransient);
@@ -1250,7 +1250,7 @@ public class CompactionStrategyManager implements INotificationConsumer
                                                                                               isTransient,
                                                                                               collector,
                                                                                               header,
-                                                                                              indexes,
+                                                                                              indexGroups,
                                                                                               lifecycleNewTracker);
         }
         finally
diff --git a/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java b/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java
index bf6c497b36..867fcd77d0 100644
--- a/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java
+++ b/src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java
@@ -240,7 +240,7 @@ public class PendingRepairHolder extends AbstractStrategyHolder
                                                        boolean isTransient,
                                                        MetadataCollector collector,
                                                        SerializationHeader header,
-                                                       Collection<Index> indexes,
+                                                       Collection<Index.Group> indexGroups,
                                                        LifecycleNewTracker lifecycleNewTracker)
     {
         Preconditions.checkArgument(repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE,
@@ -256,7 +256,7 @@ public class PendingRepairHolder extends AbstractStrategyHolder
                                                  isTransient,
                                                  collector,
                                                  header,
-                                                 indexes,
+                                                 indexGroups,
                                                  lifecycleNewTracker);
     }
 
diff --git a/src/java/org/apache/cassandra/db/compaction/Upgrader.java b/src/java/org/apache/cassandra/db/compaction/Upgrader.java
index f18cc4995c..5366878a62 100644
--- a/src/java/org/apache/cassandra/db/compaction/Upgrader.java
+++ b/src/java/org/apache/cassandra/db/compaction/Upgrader.java
@@ -17,7 +17,7 @@
  */
 package org.apache.cassandra.db.compaction;
 
-import java.util.Arrays;
+import java.util.Collections;
 import java.util.function.LongPredicate;
 
 import com.google.common.base.Throwables;
@@ -64,8 +64,8 @@ public class Upgrader
         this.controller = new UpgradeController(cfs);
 
         this.strategyManager = cfs.getCompactionStrategyManager();
-        long estimatedTotalKeys = Math.max(cfs.metadata().params.minIndexInterval, SSTableReader.getApproximateKeyCount(Arrays.asList(this.sstable)));
-        long estimatedSSTables = Math.max(1, SSTableReader.getTotalBytes(Arrays.asList(this.sstable)) / strategyManager.getMaxSSTableBytes());
+        long estimatedTotalKeys = Math.max(cfs.metadata().params.minIndexInterval, SSTableReader.getApproximateKeyCount(Collections.singletonList(this.sstable)));
+        long estimatedSSTables = Math.max(1, SSTableReader.getTotalBytes(Collections.singletonList(this.sstable)) / strategyManager.getMaxSSTableBytes());
         this.estimatedRows = (long) Math.ceil((double) estimatedTotalKeys / estimatedSSTables);
     }
 
@@ -83,8 +83,8 @@ public class Upgrader
                          .setTableMetadataRef(cfs.metadata)
                          .setMetadataCollector(sstableMetadataCollector)
                          .setSerializationHeader(SerializationHeader.make(cfs.metadata(), Sets.newHashSet(sstable)))
-                         .addDefaultComponents()
-                         .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexes(), transaction.opType())
+                         .addDefaultComponents(cfs.indexManager.listIndexGroups())
+                         .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexGroups(), transaction, cfs.metadata.get())
                          .build(transaction, cfs);
     }
 
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java
index 451a78d394..465aad3ac2 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java
@@ -159,7 +159,6 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa
 
     /**
      * Guaranteed to be called before the first call to realAppend.
-     * @param key
      */
     protected void maybeSwitchWriter(DecoratedKey key)
     {
@@ -205,7 +204,6 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa
      * Return a directory where we can expect expectedWriteSize to fit.
      *
      * @param sstables the sstables to compact
-     * @return
      */
     public Directories.DataDirectory getWriteDirectory(Iterable<SSTableReader> sstables, long estimatedWriteSize)
     {
@@ -255,9 +253,6 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa
      * - {@link SSTableWriter.Builder#setKeyCount(long)},
      * - {@link SSTableWriter.Builder#setSerializationHeader(SerializationHeader)} and,
      * - {@link SSTableWriter.Builder#setMetadataCollector(MetadataCollector)}
-     *
-     * @param descriptor
-     * @return
      */
     protected SSTableWriter.Builder<?, ?> newWriterBuilder(Descriptor descriptor)
     {
@@ -266,7 +261,7 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa
                          .setTransientSSTable(isTransient)
                          .setRepairedAt(minRepairedAt)
                          .setPendingRepair(pendingRepair)
-                         .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexes(), txn.opType())
-                         .addDefaultComponents();
+                         .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexGroups(), txn, cfs.metadata.get())
+                         .addDefaultComponents(cfs.indexManager.listIndexGroups());
     }
 }
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java
index f9858473e2..33beecf450 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java
@@ -46,13 +46,7 @@ public class DefaultCompactionWriter extends CompactionAwareWriter
         this(cfs, directories, txn, nonExpiredSSTables, false, 0);
     }
 
-    @Deprecated
-    public DefaultCompactionWriter(ColumnFamilyStore cfs, Directories directories, LifecycleTransaction txn, Set<SSTableReader> nonExpiredSSTables, boolean offline, boolean keepOriginals, int sstableLevel)
-    {
-        this(cfs, directories, txn, nonExpiredSSTables, keepOriginals, sstableLevel);
-    }
-
-    @SuppressWarnings("resource")
+    @SuppressWarnings({ "resource", "RedundantSuppression" })
     public DefaultCompactionWriter(ColumnFamilyStore cfs, Directories directories, LifecycleTransaction txn, Set<SSTableReader> nonExpiredSSTables, boolean keepOriginals, int sstableLevel)
     {
         super(cfs, directories, txn, nonExpiredSSTables, keepOriginals);
@@ -74,7 +68,7 @@ public class DefaultCompactionWriter extends CompactionAwareWriter
         MetadataCollector collector = new MetadataCollector(txn.originals(), cfs.metadata().comparator, sstableLevel);
         SerializationHeader header = SerializationHeader.make(cfs.metadata(), nonExpiredSSTables);
 
-        @SuppressWarnings("resource")
+        @SuppressWarnings({ "resource", "RedundantSuppression" })
         SSTableWriter writer = newWriterBuilder(descriptor).setMetadataCollector(collector)
                                                               .setSerializationHeader(header)
                                                               .setKeyCount(estimatedTotalKeys)
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
index 38e2d2d2c9..f5756524c3 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
@@ -51,7 +51,7 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter
         this(cfs, directories, txn, nonExpiredSSTables, maxSSTableSize, false);
     }
 
-    @SuppressWarnings("resource")
+    @SuppressWarnings({ "resource", "RedundantSuppression" })
     public MajorLeveledCompactionWriter(ColumnFamilyStore cfs,
                                         Directories directories,
                                         LifecycleTransaction txn,
@@ -67,7 +67,7 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter
     }
 
     @Override
-    @SuppressWarnings("resource")
+    @SuppressWarnings({ "resource", "RedundantSuppression" })
     public boolean realAppend(UnfilteredRowIterator partition)
     {
         AbstractRowIndexEntry rie = sstableWriter.append(partition);
@@ -97,7 +97,7 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter
         MetadataCollector collector = new MetadataCollector(txn.originals(), cfs.metadata().comparator, currentLevel);
         SerializationHeader serializationHeader = SerializationHeader.make(cfs.metadata(), txn.originals());
 
-        @SuppressWarnings("resource")
+        @SuppressWarnings({ "resource", "RedundantSuppression" })
         SSTableWriter writer = newWriterBuilder(descriptor).setKeyCount(keysPerSSTable)
                                                            .setSerializationHeader(serializationHeader)
                                                            .setMetadataCollector(collector)
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
index 834bda9e23..68706310f5 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java
@@ -98,7 +98,7 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter
         MetadataCollector collector = new MetadataCollector(allSSTables, cfs.metadata().comparator, level);
         SerializationHeader header = SerializationHeader.make(cfs.metadata(), nonExpiredSSTables);
 
-        @SuppressWarnings("resource")
+        @SuppressWarnings({ "resource", "RedundantSuppression" })
         SSTableWriter writer = newWriterBuilder(descriptor).setKeyCount(estimatedTotalKeys / estimatedSSTables)
                                                               .setMetadataCollector(collector)
                                                               .setSerializationHeader(header)
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
index 458d80a71a..da96c20363 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java
@@ -107,7 +107,7 @@ public class SplittingSizeTieredCompactionWriter extends CompactionAwareWriter
         MetadataCollector collector = new MetadataCollector(allSSTables, cfs.metadata().comparator, 0);
         SerializationHeader header = SerializationHeader.make(cfs.metadata(), nonExpiredSSTables);
 
-        @SuppressWarnings("resource")
+        @SuppressWarnings({ "resource", "RedundantSuppression" })
         SSTableWriter writer = newWriterBuilder(descriptor).setKeyCount(currentPartitionsToWrite)
                                                               .setMetadataCollector(collector)
                                                               .setSerializationHeader(header)
diff --git a/src/java/org/apache/cassandra/index/Index.java b/src/java/org/apache/cassandra/index/Index.java
index f2307b90ba..c556cb9b65 100644
--- a/src/java/org/apache/cassandra/index/Index.java
+++ b/src/java/org/apache/cassandra/index/Index.java
@@ -21,10 +21,15 @@
 package org.apache.cassandra.index;
 
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.Callable;
-import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 import org.apache.cassandra.cql3.Operator;
 import org.apache.cassandra.db.ColumnFamilyStore;
@@ -35,8 +40,8 @@ import org.apache.cassandra.db.ReadCommand;
 import org.apache.cassandra.db.ReadExecutionController;
 import org.apache.cassandra.db.RegularAndStaticColumns;
 import org.apache.cassandra.db.WriteContext;
-import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.partitions.PartitionIterator;
@@ -46,12 +51,16 @@ import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.index.internal.CollatedViewIndexBuilder;
 import org.apache.cassandra.index.transactions.IndexTransaction;
+import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.ReducingKeyIterator;
 import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+
 
 /**
  * Consisting of a top level Index interface and two sub-interfaces which handle read and write operations,
@@ -171,7 +180,7 @@ public interface Index
      */
     interface IndexBuildingSupport
     {
-        SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs, Set<Index> indexes, Collection<SSTableReader> sstables);
+        SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs, Set<Index> indexes, Collection<SSTableReader> sstables, boolean isFullRebuild);
     }
 
     /**
@@ -180,8 +189,8 @@ public interface Index
      */
     public static class CollatedViewIndexBuildingSupport implements IndexBuildingSupport
     {
-        @SuppressWarnings("resource")
-        public SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs, Set<Index> indexes, Collection<SSTableReader> sstables)
+        @SuppressWarnings({"resource", "RedundantSuppression"})
+        public SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs, Set<Index> indexes, Collection<SSTableReader> sstables, boolean isFullRebuild)
         {
             return new CollatedViewIndexBuilder(cfs, indexes, new ReducingKeyIterator(sstables), sstables);
         }
@@ -267,9 +276,10 @@ public interface Index
     public void register(IndexRegistry registry);
 
     /**
-     * If the index implementation uses a local table to store its index data this method should return a
-     * handle to it. If not, an empty Optional should be returned. Typically, this is useful for the built-in
-     * Index implementations.
+     * If the index implementation uses a local table to store its index data, this method should return a
+     * handle to it. If not, an empty {@link Optional} should be returned. This exists to support legacy 
+     * implementations, and should always be empty for indexes not belonging to a {@link SingletonIndexGroup}.
+     * 
      * @return an Optional referencing the Index's backing storage table if it has one, or Optional.empty() if not.
      */
     public Optional<ColumnFamilyStore> getBackingTable();
@@ -343,11 +353,11 @@ public interface Index
      * Get flush observer to observe partition/cell events generated by flushing SSTable (memtable flush or compaction).
      *
      * @param descriptor The descriptor of the sstable observer is requested for.
-     * @param opType The type of the operation which requests observer e.g. memtable flush or compaction.
+     * @param tracker The {@link LifecycleNewTracker} associated with the SSTable being written
      *
      * @return SSTable flush observer.
      */
-    default SSTableFlushObserver getFlushObserver(Descriptor descriptor, OperationType opType)
+    default SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker)
     {
         return null;
     }
@@ -414,6 +424,17 @@ public interface Index
      */
     public long getEstimatedResultRows();
 
+    /**
+     * Check if current index is queryable based on the index status.
+     *
+     * @param status current status of the index
+     * @return true if index should be queryable, false if index should be non-queryable
+     */
+    default boolean isQueryable(Status status)
+    {
+        return true;
+    }
+
     /*
      * Input validation
      */
@@ -425,10 +446,19 @@ public interface Index
      * cell values for each row in the update may be checked by index
      * implementations
      * @param update PartitionUpdate containing the values to be validated by registered Index implementations
-     * @throws InvalidRequestException
      */
     public void validate(PartitionUpdate update) throws InvalidRequestException;
 
+    /**
+     * Returns the SSTable-attached {@link Component}s created by this index.
+     *
+     * @return the SSTable components created by this index
+     */
+    default Set<Component> getComponents()
+    {
+        return Collections.emptySet();
+    }
+
     /*
      * Update processing
      */
@@ -444,6 +474,8 @@ public interface Index
      * @param ctx WriteContext spanning the update operation
      * @param transactionType indicates what kind of update is being performed on the base data
      *                        i.e. a write time insert/update/delete or the result of compaction
+     * @param memtable current memtable that the write goes into. It's to make sure memtable and index memtable
+     *                 are in sync.
      * @return the newly created indexer or {@code null} if the index is not interested by the update
      * (this could be because the index doesn't care about that particular partition, doesn't care about
      * that type of transaction, ...).
@@ -452,7 +484,8 @@ public interface Index
                               RegularAndStaticColumns columns,
                               int nowInSec,
                               WriteContext ctx,
-                              IndexTransaction.Type transactionType);
+                              IndexTransaction.Type transactionType,
+                              Memtable memtable);
 
     /**
      * Listener for processing events emitted during a single partition update.
@@ -483,7 +516,6 @@ public interface Index
 
         /**
          * Notification of a top level partition delete.
-         * @param deletionTime
          */
         public void partitionDelete(DeletionTime deletionTime);
 
@@ -491,7 +523,6 @@ public interface Index
          * Notification of a RangeTombstone.
          * An update of a single partition may contain multiple RangeTombstones,
          * and a notification will be passed for each of them.
-         * @param tombstone
          */
         public void rangeTombstone(RangeTombstone tombstone);
 
@@ -573,39 +604,6 @@ public interface Index
     {
     }
 
-    /**
-     * Tells whether this index supports replica fitering protection or not.
-     *
-     * Replica filtering protection might need to run the query row filter in the coordinator to detect stale results.
-     * An index implementation will be compatible with this protection mechanism if it returns the same results for the
-     * row filter as CQL will return with {@code ALLOW FILTERING} and without using the index. This means that index
-     * implementations using custom query syntax or applying transformations to the indexed data won't support it.
-     * See CASSANDRA-8272 for further details.
-     *
-     * @param rowFilter rowFilter of query to decide if it supports replica filtering protection or not
-     * @return true if this index supports replica filtering protection, false otherwise
-     */
-    default boolean supportsReplicaFilteringProtection(RowFilter rowFilter)
-    {
-        return true;
-    }
-
-    /**
-     * Return a function which performs post processing on the results of a partition range read command.
-     * In future, this may be used as a generalized mechanism for transforming results on the coordinator prior
-     * to returning them to the caller.
-     *
-     * This is used on the coordinator during execution of a range command to perform post
-     * processing of merged results obtained from the necessary replicas. This is the only way in which results are
-     * transformed in this way but this may change over time as usage is generalized.
-     * See CASSANDRA-8717 for further discussion.
-     *
-     * The function takes a PartitionIterator of the results from the replicas which has already been collated
-     * and reconciled, along with the command being executed. It returns another PartitionIterator containing the results
-     * of the transformation (which may be the same as the input if the transformation is a no-op).
-     */
-    public BiFunction<PartitionIterator, ReadCommand, PartitionIterator> postProcessorFor(ReadCommand command);
-
     /**
      * Factory method for query time search helper.
      *
@@ -621,10 +619,298 @@ public interface Index
      */
     public interface Searcher
     {
+        /**
+         * Returns the {@link ReadCommand} for which this searcher has been created.
+         *
+         * @return the base read command
+         */
+        ReadCommand command();
+
         /**
          * @param executionController the collection of OpOrder.Groups which the ReadCommand is being performed under.
          * @return partitions from the base table matching the criteria of the search.
          */
         public UnfilteredPartitionIterator search(ReadExecutionController executionController);
+
+        /**
+         * Replica filtering protection may fetch data that doesn't match query conditions.
+         *
+         * On coordinator, we need to filter the replicas' responses again.
+         * 
+         * This will not be called if {@link QueryPlan#supportsReplicaFilteringProtection(RowFilter)} returns false.
+         *
+         * @return filtered response that satisfied query conditions
+         */
+        default PartitionIterator filterReplicaFilteringProtection(PartitionIterator fullResponse)
+        {
+            return command().rowFilter().filter(fullResponse, command().metadata(), command().nowInSec());
+        }
+    }
+
+    /**
+     * Class providing grouped operations for indexes that communicate with each other.
+     *
+     * Index implementations should provide a {@code Group} implementation calling to
+     * {@link SecondaryIndexManager#registerIndex(Index, Object, Supplier)} during index registering
+     * at {@link #register(IndexRegistry)} method.
+     */
+    interface Group
+    {
+        /**
+         * Returns the indexes that are members of this group.
+         *
+         * @return the indexes that are members of this group
+         */
+        Set<Index> getIndexes();
+
+        /**
+         * Adds the specified {@link Index} as a member of this group.
+         *
+         * @param index the index to be added
+         */
+        void addIndex(Index index);
+
+        /**
+         * Removes the specified {@link Index} from the members of this group.
+         *
+         * @param index the index to be removed
+         */
+        void removeIndex(Index index);
+
+        /**
+         * Returns if this group contains the specified {@link Index}.
+         *
+         * @param index the index to be removed
+         * @return {@code true} if this group contains {@code index}, {@code false} otherwise
+         */
+        boolean containsIndex(Index index);
+
+        /**
+         * Creates an new {@code Indexer} object for updates to a given partition.
+         *
+         * @param indexSelector a predicate selecting the targeted members
+         * @param key key of the partition being modified
+         * @param columns the regular and static columns the created indexer will have to deal with.
+         * This can be empty as an update might only contain partition, range and row deletions, but
+         * the indexer is guaranteed to not get any cells for a column that is not part of {@code columns}.
+         * @param nowInSec current time of the update operation
+         * @param ctx WriteContext spanning the update operation
+         * @param transactionType indicates what kind of update is being performed on the base data
+         *                        i.e. a write time insert/update/delete or the result of compaction
+         * @param memtable the {@link Memtable} to which the updates are being applied or {@code null}
+         *                 if the source of the updates is an existing {@link SSTable}
+         *
+         * @return the newly created indexer or {@code null} if the index is not interested by the update
+         * (this could be because the index doesn't care about that particular partition, doesn't care about
+         * that type of transaction, ...).
+         */
+        Indexer indexerFor(Predicate<Index> indexSelector,
+                           DecoratedKey key,
+                           RegularAndStaticColumns columns,
+                           int nowInSec,
+                           WriteContext ctx,
+                           IndexTransaction.Type transactionType,
+                           Memtable memtable);
+
+        /**
+         * Returns a new {@link QueryPlan} for the specified {@link RowFilter}, or {@code null} if none of the indexes in
+         * this group supports the expression in the row filter.
+         *
+         * @param rowFilter a row filter
+         * @return a new query plan for the specified {@link RowFilter} if it's supported, {@code null} otherwise
+         */
+        @Nullable
+        QueryPlan queryPlanFor(RowFilter rowFilter);
+
+        /**
+         * Get flush observer to observe partition/cell events generated by flushing SSTable (memtable flush or compaction).
+         *
+         * @param descriptor The descriptor of the sstable observer is requested for.
+         * @param tracker The {@link LifecycleNewTracker} associated with the SSTable being written
+         * @param tableMetadata The immutable metadata of the table at the moment the SSTable is flushed
+         *
+         * @return SSTable flush observer.
+         */
+        SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata);
+
+        /**
+         * @param type index transaction type
+         * @return true if index will be able to handle given index transaction type
+         */
+        default boolean handles(IndexTransaction.Type type)
+        {
+            return true;
+        }
+
+        /**
+         * Called when the table associated with this group has been invalidated. Implementations
+         * should dispose of any resources tied to the lifecycle of the {@link Group}.
+         */
+        default void invalidate() { }
+
+        /**
+         * Returns the SSTable-attached {@link Component}s created by this index group.
+         *
+         * @return the SSTable components created by this group
+         */
+        Set<Component> getComponents();
+    }
+
+    /**
+     * Specifies a set of compatible indexes to be used with a query according to its {@link RowFilter}, ignoring data
+     * ranges, limits, etc. All the indexes in it should belong to the same {@link Group}.
+     * <p>
+     * It's created by {@link Group#queryPlanFor} from the {@link RowFilter} that is common to all the subcommands of a
+     * user query's {@link ReadCommand}, so it can be reused by those subcommands along the cluster nodes. The
+     * {@link #searcherFor(ReadCommand)} method provides the {@link Searcher} object to read the index for each
+     * particular (sub)command.
+     */
+    interface QueryPlan extends Comparable<QueryPlan>
+    {
+        /**
+         * Returns the indexes selected by this query plan, all of them belonging to the same {@link Group}.
+         *
+         * It should never be empty.
+         *
+         * @return the indexes selected by this query plan, which is never empty
+         */
+        Set<Index> getIndexes();
+
+        /**
+         * Returns the first index in this plan.
+         *
+         * @return the first index
+         */
+        @Nonnull
+        default Index getFirst()
+        {
+            return getIndexes().iterator().next();
+        }
+
+        /**
+         * Return an estimate of the number of results this plan is expected to return for any given {@link ReadCommand}
+         * that it can be used to answer. Used by  {@link SecondaryIndexManager#getBestIndexQueryPlanFor(RowFilter)}
+         * to determine the {@link Group} with the most selective plan for a given {@link RowFilter}.
+         * Additionally, this is also used by StorageProxy.estimateResultsPerRange to calculate the initial concurrency
+         * factor for range requests
+         *
+         * @return the estimated average number of results a Searcher may return for any given command
+         */
+        default long getEstimatedResultRows()
+        {
+            // CQL only supports AND expressions, so the estimated number of results for multiple indexes will be the
+            // the lowest of the estimates for each index
+            return getIndexes().stream()
+                               .mapToLong(Index::getEstimatedResultRows)
+                               .min()
+                               .orElseThrow(AssertionError::new); // registered groups are never empty
+        }
+
+        /**
+         * Used to determine whether to estimate initial concurrency during remote range reads. Default is true, each
+         * implementation must override this method if they choose a different strategy.
+         *
+         * @return true if the {@link QueryPlan} should estimate initial concurrency, false otherwise
+         */
+        default boolean shouldEstimateInitialConcurrency()
+        {
+            return true;
+        }
+
+        @Override
+        default int compareTo(QueryPlan other)
+        {
+            // initially, we prefer the plan with less estimated results
+            int results = Long.compare(getEstimatedResultRows(), other.getEstimatedResultRows());
+            if (results != 0)
+                return results;
+
+            // In case of having the same number of estimated results, we favour the plan that involves more indexes.
+            // This way, we honour the possible absence of ALLOW FILTERING in the CQL query. Also, this criteria should
+            // not break the transitivity of this method because the estimated number of results for a plan is the
+            // minimum of the estimates of its members.
+            return Integer.compare(getIndexes().size(), other.getIndexes().size());
+        }
+
+        /**
+         * Used to validate the various parameters of a supplied {@link ReadCommand} against the indexes in this plan.
+         *
+         * @param command a ReadCommand whose parameters are to be verified
+         * @throws InvalidRequestException if the details of the command fail to meet the validation rules of the
+         * indexes in the query plan
+         */
+        default void validate(ReadCommand command) throws InvalidRequestException
+        {
+            getIndexes().forEach(i -> i.validate(command));
+        }
+
+        /**
+         * Factory method for query time search helper.
+         *
+         * @param command the read command being executed
+         * @return an Searcher with which to perform the supplied command
+         */
+        Searcher searcherFor(ReadCommand command);
+
+        /**
+         * Return a function which performs post processing on the results of a partition range read command.
+         * In future, this may be used as a generalized mechanism for transforming results on the coordinator prior
+         * to returning them to the caller.
+         *
+         * This is used on the coordinator during execution of a range command to perform post
+         * processing of merged results obtained from the necessary replicas. This is the only way in which results are
+         * transformed in this way but this may change over time as usage is generalized.
+         * See CASSANDRA-8717 for further discussion.
+         *
+         * The function takes a PartitionIterator of the results from the replicas which has already been collated
+         * and reconciled, along with the command being executed. It returns another PartitionIterator containing the results
+         * of the transformation (which may be the same as the input if the transformation is a no-op).
+         */
+        default Function<PartitionIterator, PartitionIterator> postProcessor()
+        {
+            return partitions -> partitions;
+        }
+
+        /**
+         * Transform an initial {@link RowFilter} into the filter that will still need to applied to a set of Rows after
+         * the index has performed it's initial scan.
+         *
+         * Used in {@link ReadCommand#executeLocally(ReadExecutionController)} to reduce the amount of filtering performed on the
+         * results of the index query.
+         *
+         * @return the (hopefully) reduced filter that would still need to be applied after
+         *         the index was used to narrow the initial result set
+         */
+        RowFilter postIndexQueryFilter();
+
+
+        /**
+         * Tells whether this index supports replica fitering protection or not.
+         *
+         * Replica filtering protection might need to run the query row filter in the coordinator to detect stale results.
+         * An index implementation will be compatible with this protection mechanism if it returns the same results for the
+         * row filter as CQL will return with {@code ALLOW FILTERING} and without using the index. This means that index
+         * implementations using custom query syntax or applying transformations to the indexed data won't support it.
+         * See CASSANDRA-8272 for further details.
+         *
+         * @param rowFilter rowFilter of query to decide if it supports replica filtering protection or not
+         * @return true if this index supports replica filtering protection, false otherwise
+         */
+        default boolean supportsReplicaFilteringProtection(RowFilter rowFilter)
+        {
+            return true;
+        }
+    }
+
+    /*
+     * Status of index used to determine queryability
+     */
+    enum Status
+    {
+        UNKNOWN,
+        FULL_REBUILD_STARTED,
+        BUILD_FAILED,
+        BUILD_SUCCEEDED,
+        DROPPED
     }
 }
diff --git a/src/java/org/apache/cassandra/index/IndexRegistry.java b/src/java/org/apache/cassandra/index/IndexRegistry.java
index 0cf1cbb6c2..5bce303e6c 100644
--- a/src/java/org/apache/cassandra/index/IndexRegistry.java
+++ b/src/java/org/apache/cassandra/index/IndexRegistry.java
@@ -23,19 +23,27 @@ package org.apache.cassandra.index;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.Callable;
-import java.util.function.BiFunction;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+
+import javax.annotation.Nullable;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.Operator;
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.BytesType;
-import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.cassandra.index.transactions.IndexTransaction;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableFlushObserver;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.TableMetadata;
@@ -56,19 +64,20 @@ public interface IndexRegistry
     public static final IndexRegistry EMPTY = new IndexRegistry()
     {
         @Override
-        public void unregisterIndex(Index index)
+        public void registerIndex(Index index, Object groupKey, Supplier<Index.Group> groupSupplier)
         {
         }
 
         @Override
-        public void registerIndex(Index index)
+        public Collection<Index> listIndexes()
         {
+            return Collections.emptyList();
         }
 
         @Override
-        public Collection<Index> listIndexes()
+        public Collection<Index.Group> listIndexGroups()
         {
-            return Collections.emptyList();
+            return Collections.emptySet();
         }
 
         @Override
@@ -97,7 +106,7 @@ public interface IndexRegistry
      */
     public static final IndexRegistry NON_DAEMON = new IndexRegistry()
     {
-        Index index = new Index()
+        final Index index = new Index()
         {
             public Callable<?> getInitializationTask()
             {
@@ -173,27 +182,70 @@ public interface IndexRegistry
             {
             }
 
-            public Indexer indexerFor(DecoratedKey key, RegularAndStaticColumns columns, int nowInSec, WriteContext ctx, IndexTransaction.Type transactionType)
+            public Indexer indexerFor(DecoratedKey key, RegularAndStaticColumns columns, int nowInSec, WriteContext ctx, IndexTransaction.Type transactionType, Memtable memtable)
             {
                 return null;
             }
 
-            public BiFunction<PartitionIterator, ReadCommand, PartitionIterator> postProcessorFor(ReadCommand command)
+            public Searcher searcherFor(ReadCommand command)
             {
                 return null;
             }
+        };
 
-            public Searcher searcherFor(ReadCommand command)
+        final Index.Group group = new Index.Group()
+        {
+            @Override
+            public Set<Index> getIndexes()
+            {
+                return Collections.singleton(index);
+            }
+
+            @Override
+            public void addIndex(Index index)
+            {
+            }
+
+            @Override
+            public void removeIndex(Index index)
+            {
+            }
+
+            @Override
+            public boolean containsIndex(Index i)
+            {
+                return index == i;
+            }
+
+            @Nullable
+            @Override
+            public Index.Indexer indexerFor(Predicate<Index> indexSelector, DecoratedKey key, RegularAndStaticColumns columns, int nowInSec, WriteContext ctx, IndexTransaction.Type transactionType, Memtable memtable)
             {
                 return null;
             }
-        };
 
-        public void registerIndex(Index index)
-        {
-        }
+            @Nullable
+            @Override
+            public Index.QueryPlan queryPlanFor(RowFilter rowFilter)
+            {
+                return null;
+            }
 
-        public void unregisterIndex(Index index)
+            @Nullable
+            @Override
+            public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata)
+            {
+                return null;
+            }
+
+            @Override
+            public Set<Component> getComponents()
+            {
+                return null;
+            }
+        };
+
+        public void registerIndex(Index index, Object groupKey, Supplier<Index.Group> groupSupplier)
         {
         }
 
@@ -207,6 +259,12 @@ public interface IndexRegistry
             return Collections.singletonList(index);
         }
 
+        @Override
+        public Collection<Index.Group> listIndexGroups()
+        {
+            return Collections.singletonList(group);
+        }
+
         public Optional<Index> getBestIndexFor(RowFilter.Expression expression)
         {
             return Optional.empty();
@@ -214,12 +272,15 @@ public interface IndexRegistry
 
         public void validate(PartitionUpdate update)
         {
-
         }
     };
 
-    void registerIndex(Index index);
-    void unregisterIndex(Index index);
+    default void registerIndex(Index index)
+    {
+        registerIndex(index, index, () -> new SingletonIndexGroup(index));
+    }
+    public void registerIndex(Index index, Object groupKey, Supplier<Index.Group> groupSupplier);
+    Collection<Index.Group> listIndexGroups();
 
     Index getIndex(IndexMetadata indexMetadata);
     Collection<Index> listIndexes();
diff --git a/src/java/org/apache/cassandra/index/SecondaryIndexManager.java b/src/java/org/apache/cassandra/index/SecondaryIndexManager.java
index 4c412a2887..1aec1da481 100644
--- a/src/java/org/apache/cassandra/index/SecondaryIndexManager.java
+++ b/src/java/org/apache/cassandra/index/SecondaryIndexManager.java
@@ -24,14 +24,21 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import javax.annotation.Nullable;
+
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Joiner;
 import com.google.common.base.Strings;
-import com.google.common.collect.*;
-import com.google.common.primitives.Longs;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.FutureCallback;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
@@ -149,6 +156,11 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
      */
     private final Map<String, Index> writableIndexes = Maps.newConcurrentMap();
 
+    /**
+     * The groups of all the registered indexes
+     */
+    private final Map<Object, Index.Group> indexGroups = Maps.newConcurrentMap();
+
     /**
      * The count of pending index builds for each index.
      */
@@ -212,9 +224,15 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
 
         markIndexesBuilding(ImmutableSet.of(index), true, isNewCF);
 
+        return buildIndex(index);
+    }
+
+    @VisibleForTesting
+    public Future<Void> buildIndex(final Index index)
+    {
         FutureTask<?> initialBuildTask = null;
         // if the index didn't register itself, we can probably assume that no initialization needs to happen
-        if (indexes.containsKey(indexDef.name))
+        if (indexes.containsKey(index.getIndexMetadata().name))
         {
             try
             {
@@ -281,7 +299,23 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
     {
         return queryableIndexes.contains(index.getIndexMetadata().name);
     }
-    
+
+    /**
+     * Throws an {@link IndexNotAvailableException} if any of the indexes in the specified {@link Index.QueryPlan} is
+     * not queryable, as it's defined by {@link #isIndexQueryable(Index)}.
+     *
+     * @param queryPlan a query plan
+     * @throws IndexNotAvailableException if the query plan has any index that is not queryable
+     */
+    public void checkQueryability(Index.QueryPlan queryPlan)
+    {
+        for (Index index : queryPlan.getIndexes())
+        {
+            if (!isIndexQueryable(index))
+                throw new IndexNotAvailableException(index);
+        }
+    }
+
     /**
      * Checks if the specified index is writable.
      *
@@ -490,7 +524,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
         {
             logger.info("Submitting index {} of {} for data in {}",
                         isFullRebuild ? "recovery" : "build",
-                        indexes.stream().map(i -> i.getIndexMetadata().name).collect(Collectors.joining(",")),
+                        commaSeparated(indexes),
                         sstables.stream().map(SSTableReader::toString).collect(Collectors.joining(",")));
 
             // Group all building tasks
@@ -508,7 +542,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
             List<Future<?>> futures = new ArrayList<>(byType.size());
             byType.forEach((buildingSupport, groupedIndexes) ->
                            {
-                               SecondaryIndexBuilder builder = buildingSupport.getIndexBuildTask(baseCfs, groupedIndexes, sstables);
+                               SecondaryIndexBuilder builder = buildingSupport.getIndexBuildTask(baseCfs, groupedIndexes, sstables, isFullRebuild);
                                final AsyncPromise<Object> build = new AsyncPromise<>();
                                CompactionManager.instance.submitIndexBuild(builder).addCallback(new FutureCallback()
                                {
@@ -643,7 +677,10 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
                             AtomicInteger counter = inProgressBuilds.computeIfAbsent(indexName, ignored -> new AtomicInteger(0));
 
                             if (isFullRebuild)
+                            {
                                 needsFullRebuild.remove(indexName);
+                                makeIndexNonQueryable(index, Index.Status.FULL_REBUILD_STARTED);
+                            }
 
                             if (counter.getAndIncrement() == 0 && DatabaseDescriptor.isDaemonInitialized() && !isNewCF)
                                 SystemKeyspace.setIndexRemoved(keyspaceName, indexName);
@@ -661,14 +698,8 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
     {
         String indexName = index.getIndexMetadata().name;
         if (isFullRebuild)
-        {
-            if (queryableIndexes.add(indexName))
-                logger.info("Index [{}] became queryable after successful build.", indexName);
-
-            if (writableIndexes.put(indexName, index) == null)
-                logger.info("Index [{}] became writable after successful build.", indexName);
-        }
-
+            makeIndexQueryable(index, Index.Status.BUILD_SUCCEEDED);
+        
         AtomicInteger counter = inProgressBuilds.get(indexName);
         if (counter != null)
         {
@@ -748,8 +779,10 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
         if (indexDef.isCustom())
         {
             assert indexDef.options != null;
-            String className = indexDef.options.get(IndexTarget.CUSTOM_INDEX_OPTION_NAME);
+            // Find any aliases to the fully qualified index class name:
+            String className = IndexMetadata.expandAliases(indexDef.options.get(IndexTarget.CUSTOM_INDEX_OPTION_NAME));
             assert !Strings.isNullOrEmpty(className);
+
             try
             {
                 Class<? extends Index> indexClass = FBUtilities.classForName(className, "Index");
@@ -784,6 +817,9 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
         markAllIndexesRemoved();
         if (dropData)
             invalidateAllIndexesBlocking();
+
+        // TODO: Determine whether "dropData" should guard this or be passed to Group#invalidate()
+        indexGroups.forEach((key, group) -> group.invalidate());
     }
 
     @VisibleForTesting
@@ -927,12 +963,14 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
 
                     try (UnfilteredRowIterator partition = page.next())
                     {
-                        Set<Index.Indexer> indexers = indexes.stream()
-                                                             .map(index -> index.indexerFor(key,
-                                                                                            partition.columns(),
-                                                                                            nowInSec,
-                                                                                            ctx,
-                                                                                            IndexTransaction.Type.UPDATE))
+                        Set<Index.Indexer> indexers = indexGroups.values().stream()
+                                                             .map(g -> g.indexerFor(indexes::contains,
+                                                                                    key,
+                                                                                    partition.columns(),
+                                                                                    nowInSec,
+                                                                                    ctx,
+                                                                                    IndexTransaction.Type.UPDATE,
+                                                                                    null))
                                                              .filter(Objects::nonNull)
                                                              .collect(Collectors.toSet());
 
@@ -1033,6 +1071,9 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
      */
     public void deletePartition(UnfilteredRowIterator partition, int nowInSec)
     {
+        if (!handles(IndexTransaction.Type.CLEANUP))
+            return;
+
         // we need to acquire memtable lock because secondary index deletion may
         // cause a race (see CASSANDRA-3712). This is done internally by the
         // index transaction when it commits
@@ -1080,15 +1121,14 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
      * ReadOrderGroup, or an estimate of the result size from an average index query.
      *
      * @param rowFilter RowFilter of the command to be executed
-     * @return an Index instance, ready to use during execution of the command, or null if none
-     * of the registered indexes can support the command.
+     * @return the best available index query plan for the row filter, or {@code null} if none of the registered indexes
+     * can support the command.
      */
-    public Index getBestIndexFor(RowFilter rowFilter)
+    public Index.QueryPlan getBestIndexQueryPlanFor(RowFilter rowFilter)
     {
         if (indexes.isEmpty() || rowFilter.isEmpty())
             return null;
 
-        Set<Index> searchableIndexes = new HashSet<>();
         for (RowFilter.Expression expression : rowFilter)
         {
             if (expression.isCustom())
@@ -1098,41 +1138,49 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
                 RowFilter.CustomExpression customExpression = (RowFilter.CustomExpression) expression;
                 logger.trace("Command contains a custom index expression, using target index {}", customExpression.getTargetIndex().name);
                 Tracing.trace("Command contains a custom index expression, using target index {}", customExpression.getTargetIndex().name);
-                return indexes.get(customExpression.getTargetIndex().name);
-            }
-            else if (!expression.isUserDefined())
-            {
-                indexes.values().stream()
-                       .filter(index -> index.supportsExpression(expression.column(), expression.operator()))
-                       .forEach(searchableIndexes::add);
+                Index.Group group = getIndexGroup(customExpression.getTargetIndex());
+                return group == null ? null : group.queryPlanFor(rowFilter);
             }
         }
 
-        if (searchableIndexes.isEmpty())
+        Set<Index.QueryPlan> queryPlans = indexGroups.values()
+                                                     .stream()
+                                                     .map(g -> g.queryPlanFor(rowFilter))
+                                                     .filter(Objects::nonNull)
+                                                     .collect(Collectors.toSet());
+
+        if (queryPlans.isEmpty())
         {
             logger.trace("No applicable indexes found");
             Tracing.trace("No applicable indexes found");
             return null;
         }
 
-        Index selected = searchableIndexes.size() == 1
-                         ? Iterables.getOnlyElement(searchableIndexes)
-                         : searchableIndexes.stream()
-                                            .min((a, b) -> Longs.compare(a.getEstimatedResultRows(),
-                                                                         b.getEstimatedResultRows()))
-                                            .orElseThrow(() -> new AssertionError("Could not select most selective index"));
+        // find the best plan
+        Index.QueryPlan selected = queryPlans.size() == 1
+                                   ? Iterables.getOnlyElement(queryPlans)
+                                   : queryPlans.stream()
+                                               .min(Comparator.naturalOrder())
+                                               .orElseThrow(() -> new AssertionError("Could not select most selective index"));
 
         // pay for an additional threadlocal get() rather than build the strings unnecessarily
         if (Tracing.isTracing())
         {
             Tracing.trace("Index mean cardinalities are {}. Scanning with {}.",
-                          searchableIndexes.stream().map(i -> i.getIndexMetadata().name + ':' + i.getEstimatedResultRows())
-                                           .collect(Collectors.joining(",")),
-                          selected.getIndexMetadata().name);
+                          queryPlans.stream()
+                                    .map(p -> commaSeparated(p.getIndexes()) + ':' + p.getEstimatedResultRows())
+                                    .collect(Collectors.joining(",")),
+                          commaSeparated(selected.getIndexes()));
         }
         return selected;
     }
 
+    private static String commaSeparated(Collection<Index> indexes)
+    {
+        return indexes.stream().map(i -> i.getIndexMetadata().name).collect(Collectors.joining(","));
+    }
+
+
     public Optional<Index> getBestIndexFor(RowFilter.Expression expression)
     {
         return indexes.values().stream().filter((i) -> i.supportsExpression(expression.column(), expression.operator())).findFirst();
@@ -1157,23 +1205,50 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
     /*
      * IndexRegistry methods
      */
-
-    public void registerIndex(Index index)
+    public void registerIndex(Index index, Object groupKey, Supplier<Index.Group> groupSupplier)
     {
         String name = index.getIndexMetadata().name;
         indexes.put(name, index);
         logger.trace("Registered index {}", name);
-    }
 
-    public void unregisterIndex(Index index)
-    {
-        unregisterIndex(index.getIndexMetadata().name);
+        // instantiate and add the index group if it hasn't been already added
+        Index.Group group = indexGroups.computeIfAbsent(groupKey, k -> groupSupplier.get());
+
+        // add the created index to its group if it is not a singleton group
+        if (!(group instanceof SingletonIndexGroup))
+        {
+            if (index.getBackingTable().isPresent())
+                throw new InvalidRequestException("Indexes belonging to a group of indexes shouldn't have a backing table");
+
+            group.addIndex(index);
+        }
     }
 
     private Index unregisterIndex(String name)
     {
         Index removed = indexes.remove(name);
         logger.trace(removed == null ? "Index {} was not registered" : "Removed index {} from registry", name);
+
+        if (removed != null)
+        {
+            // Remove the index from any non-singleton groups...
+            for (Index.Group group : listIndexGroups())
+            {
+                if (!(group instanceof SingletonIndexGroup) && group.containsIndex(removed))
+                {
+                    group.removeIndex(removed);
+
+                    if (group.getIndexes().isEmpty())
+                    {
+                        indexGroups.remove(group);
+                    }
+                }
+            }
+
+            // ...and remove singleton groups entirely.
+            indexGroups.remove(removed);
+        }
+
         return removed;
     }
 
@@ -1187,6 +1262,42 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
         return ImmutableSet.copyOf(indexes.values());
     }
 
+    public Set<Index.Group> listIndexGroups()
+    {
+        return ImmutableSet.copyOf(indexGroups.values());
+    }
+
+    public Index.Group getIndexGroup(Object key)
+    {
+        return indexGroups.get(key);
+    }
+
+    /**
+     * Returns the {@link Index.Group} the specified index belongs to, as specified during registering with
+     * {@link #registerIndex(Index, Object, Supplier)}.
+     *
+     * @param metadata the index metadata
+     * @return the group the index belongs to, or {@code null} if the index is not registered or if it hasn't been
+     * associated to any group
+     */
+    @Nullable
+    public Index.Group getIndexGroup(IndexMetadata metadata)
+    {
+        Index index = getIndex(metadata);
+        return index == null ? null : getIndexGroup(index);
+    }
+
+    @VisibleForTesting
+    public boolean needsFullRebuild(String index)
+    {
+        return needsFullRebuild.contains(index);
+    }
+
+    public Index.Group getIndexGroup(Index index)
+    {
+        return indexGroups.values().stream().filter(g -> g.containsIndex(index)).findAny().orElse(null);
+    }
+
     /*
      * Handling of index updates.
      * Implementations of the various IndexTransaction interfaces, for keeping indexes in sync with base data
@@ -1196,23 +1307,28 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
     /**
      * Transaction for updates on the write path.
      */
-    public UpdateTransaction newUpdateTransaction(PartitionUpdate update, WriteContext ctx, int nowInSec)
+    public UpdateTransaction newUpdateTransaction(PartitionUpdate update, WriteContext ctx, int nowInSec, Memtable memtable)
     {
         if (!hasIndexes())
             return UpdateTransaction.NO_OP;
 
-        ArrayList<Index.Indexer> idxrs = new ArrayList<>();
-        for (Index i : writableIndexes.values())
-        {
-            Index.Indexer idxr = i.indexerFor(update.partitionKey(), update.columns(), nowInSec, ctx, IndexTransaction.Type.UPDATE);
-            if (idxr != null)
-                idxrs.add(idxr);
-        }
+        Index.Indexer[] indexers = listIndexGroups().stream()
+                                          .map(g -> g.indexerFor(writableIndexSelector(),
+                                                                 update.partitionKey(),
+                                                                 update.columns(),
+                                                                 nowInSec,
+                                                                 ctx,
+                                                                 IndexTransaction.Type.UPDATE,
+                                                                 memtable))
+                                          .filter(Objects::nonNull)
+                                          .toArray(Index.Indexer[]::new);
+
+        return indexers.length == 0 ? UpdateTransaction.NO_OP : new WriteTimeTransaction(indexers);
+    }
 
-        if (idxrs.size() == 0)
-            return UpdateTransaction.NO_OP;
-        else
-            return new WriteTimeTransaction(idxrs.toArray(new Index.Indexer[idxrs.size()]));
+    private Predicate<Index> writableIndexSelector()
+    {
+        return index -> writableIndexes.containsKey(index.getIndexMetadata().name);
     }
 
     /**
@@ -1224,7 +1340,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
                                                           int nowInSec)
     {
         // the check for whether there are any registered indexes is already done in CompactionIterator
-        return new IndexGCTransaction(key, regularAndStaticColumns, keyspace, versions, nowInSec, writableIndexes.values());
+        return new IndexGCTransaction(key, regularAndStaticColumns, keyspace, versions, nowInSec, listIndexGroups(), writableIndexSelector());
     }
 
     /**
@@ -1237,7 +1353,21 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
         if (!hasIndexes())
             return CleanupTransaction.NO_OP;
 
-        return new CleanupGCTransaction(key, regularAndStaticColumns, keyspace, nowInSec, writableIndexes.values());
+        return new CleanupGCTransaction(key, regularAndStaticColumns, keyspace, nowInSec, listIndexGroups(), writableIndexSelector());
+    }
+
+    /**
+     * @param type index transaction type
+     * @return true if at least one of the indexes will be able to handle given index transaction type
+     */
+    public boolean handles(IndexTransaction.Type type)
+    {
+        for (Index.Group group : indexGroups.values())
+        {
+            if (group.handles(type))
+                return true;
+        }
+        return false;
     }
 
     /**
@@ -1353,7 +1483,8 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
         private final Keyspace keyspace;
         private final int versions;
         private final int nowInSec;
-        private final Collection<Index> indexes;
+        private final Collection<Index.Group> indexGroups;
+        private final Predicate<Index> writableIndexSelector;
 
         private Row[] rows;
 
@@ -1362,14 +1493,16 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
                                    Keyspace keyspace,
                                    int versions,
                                    int nowInSec,
-                                   Collection<Index> indexes)
+                                   Collection<Index.Group> indexGroups,
+                                   Predicate<Index> writableIndexSelector)
         {
             this.key = key;
             this.columns = columns;
             this.keyspace = keyspace;
             this.versions = versions;
-            this.indexes = indexes;
+            this.indexGroups = indexGroups;
             this.nowInSec = nowInSec;
+            this.writableIndexSelector = writableIndexSelector;
         }
 
         public void start()
@@ -1430,9 +1563,9 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
 
             try (WriteContext ctx = keyspace.getWriteHandler().createContextForIndexing())
             {
-                for (Index index : indexes)
+                for (Index.Group group : indexGroups)
                 {
-                    Index.Indexer indexer = index.indexerFor(key, columns, nowInSec, ctx, Type.COMPACTION);
+                    Index.Indexer indexer = group.indexerFor(writableIndexSelector, key, columns, nowInSec, ctx, Type.COMPACTION, null);
                     if (indexer == null)
                         continue;
 
@@ -1458,7 +1591,8 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
         private final RegularAndStaticColumns columns;
         private final Keyspace keyspace;
         private final int nowInSec;
-        private final Collection<Index> indexes;
+        private final Collection<Index.Group> indexGroups;
+        private final Predicate<Index> writableIndexSelector;
 
         private Row row;
         private DeletionTime partitionDelete;
@@ -1467,13 +1601,15 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
                                      RegularAndStaticColumns columns,
                                      Keyspace keyspace,
                                      int nowInSec,
-                                     Collection<Index> indexes)
+                                     Collection<Index.Group> indexGroups,
+                                     Predicate<Index> writableIndexSelector)
         {
             this.key = key;
             this.columns = columns;
             this.keyspace = keyspace;
-            this.indexes = indexes;
+            this.indexGroups = indexGroups;
             this.nowInSec = nowInSec;
+            this.writableIndexSelector = writableIndexSelector;
         }
 
         public void start()
@@ -1497,9 +1633,9 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
 
             try (WriteContext ctx = keyspace.getWriteHandler().createContextForIndexing())
             {
-                for (Index index : indexes)
+                for (Index.Group group : indexGroups)
                 {
-                    Index.Indexer indexer = index.indexerFor(key, columns, nowInSec, ctx, Type.CLEANUP);
+                    Index.Indexer indexer = group.indexerFor(writableIndexSelector, key, columns, nowInSec, ctx, Type.CLEANUP, null);
                     if (indexer == null)
                         continue;
 
@@ -1572,4 +1708,30 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
         shutdown(asyncExecutor, blockingExecutor);
         awaitTermination(timeout, units, asyncExecutor, blockingExecutor);
     }
+
+    public void makeIndexNonQueryable(Index index, Index.Status status)
+    {
+        String name = index.getIndexMetadata().name;
+        if (indexes.get(name) == index)
+        {
+            if (!index.isQueryable(status))
+                queryableIndexes.remove(name);
+        }
+    }
+
+    public void makeIndexQueryable(Index index, Index.Status status)
+    {
+        String name = index.getIndexMetadata().name;
+        if (indexes.get(name) == index)
+        {
+            if (index.isQueryable(status))
+            {
+                if (queryableIndexes.add(name))
+                    logger.info("Index [{}] became queryable after successful build.", name);
+            }
+
+            if (writableIndexes.put(name, index) == null)
+                logger.info("Index [{}] became writable after successful build.", name);
+        }
+    }
 }
diff --git a/src/java/org/apache/cassandra/index/SingletonIndexGroup.java b/src/java/org/apache/cassandra/index/SingletonIndexGroup.java
new file mode 100644
index 0000000000..0a1a394e45
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/SingletonIndexGroup.java
@@ -0,0 +1,114 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.cassandra.index;
+
+import java.util.Collections;
+import java.util.Set;
+import java.util.function.Predicate;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.WriteContext;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.index.transactions.IndexTransaction;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * An {@link Index.Group} containing a single {@link Index}, to which it just delegates the calls.
+ */
+public class SingletonIndexGroup implements Index.Group
+{
+    private final Index delegate;
+    private final Set<Index> indexes;
+
+    protected SingletonIndexGroup(Index delegate)
+    {
+        this.delegate = delegate;
+        this.indexes = Collections.singleton(delegate);
+    }
+
+    @Override
+    public Set<Index> getIndexes()
+    {
+        return indexes;
+    }
+
+    public Index getIndex()
+    {
+        return delegate;
+    }
+
+    @Override
+    public void addIndex(Index index)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void removeIndex(Index index)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean containsIndex(Index index)
+    {
+        return index.equals(delegate);
+    }
+
+    @Override
+    public Index.Indexer indexerFor(Predicate<Index> indexSelector,
+                                    DecoratedKey key,
+                                    RegularAndStaticColumns columns,
+                                    int nowInSec,
+                                    WriteContext ctx,
+                                    IndexTransaction.Type transactionType,
+                                    Memtable memtable)
+    {
+        return indexSelector.test(delegate)
+               ? delegate.indexerFor(key, columns, nowInSec, ctx, transactionType, memtable)
+               : null;
+    }
+
+    @Override
+    public Index.QueryPlan queryPlanFor(RowFilter rowFilter)
+    {
+        return SingletonIndexQueryPlan.create(delegate, rowFilter);
+    }
+
+    @Override
+    public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata)
+    {
+        return delegate.getFlushObserver(descriptor, tracker);
+    }
+
+    @Override
+    public Set<Component> getComponents()
+    {
+        return delegate.getComponents();
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/SingletonIndexQueryPlan.java b/src/java/org/apache/cassandra/index/SingletonIndexQueryPlan.java
new file mode 100644
index 0000000000..b475cee145
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/SingletonIndexQueryPlan.java
@@ -0,0 +1,87 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.cassandra.index;
+
+import java.util.Collections;
+import java.util.Set;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.db.ReadCommand;
+import org.apache.cassandra.db.filter.RowFilter;
+
+public class SingletonIndexQueryPlan implements Index.QueryPlan
+{
+    private final Index index;
+    private final Set<Index> indexes;
+    private final RowFilter postIndexFilter;
+
+    protected SingletonIndexQueryPlan(Index index, RowFilter postIndexFilter)
+    {
+        this.index = index;
+        this.indexes = Collections.singleton(index);
+        this.postIndexFilter = postIndexFilter;
+    }
+
+    @Nullable
+    protected static SingletonIndexQueryPlan create(Index index, RowFilter rowFilter)
+    {
+        for (RowFilter.Expression e : rowFilter.getExpressions())
+        {
+            if (index.supportsExpression(e.column(), e.operator()))
+                return new SingletonIndexQueryPlan(index, index.getPostIndexQueryFilter(rowFilter));
+        }
+
+        return null;
+    }
+
+    @Override
+    public Set<Index> getIndexes()
+    {
+        return indexes;
+    }
+
+    @Override
+    @Nonnull
+    public Index getFirst()
+    {
+        return index;
+    }
+
+    @Override
+    public long getEstimatedResultRows()
+    {
+        return index.getEstimatedResultRows();
+    }
+
+    @Override
+    public Index.Searcher searcherFor(ReadCommand command)
+    {
+        return index.searcherFor(command);
+    }
+
+    @Override
+    public RowFilter postIndexQueryFilter()
+    {
+        return postIndexFilter;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/internal/CassandraIndex.java b/src/java/org/apache/cassandra/index/internal/CassandraIndex.java
index 0aac15d994..3716c23951 100644
--- a/src/java/org/apache/cassandra/index/internal/CassandraIndex.java
+++ b/src/java/org/apache/cassandra/index/internal/CassandraIndex.java
@@ -24,7 +24,6 @@ import java.nio.ByteBuffer;
 import java.util.*;
 import java.util.concurrent.Callable;
 import java.util.concurrent.Future;
-import java.util.function.BiFunction;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 
@@ -33,9 +32,6 @@ import com.google.common.collect.ImmutableSet;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.cassandra.schema.TableMetadata;
-import org.apache.cassandra.schema.TableMetadataRef;
-import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.cql3.Operator;
 import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.db.*;
@@ -45,7 +41,7 @@ import org.apache.cassandra.db.lifecycle.SSTableSet;
 import org.apache.cassandra.db.lifecycle.View;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.CollectionType;
-import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.*;
 import org.apache.cassandra.dht.LocalPartitioner;
@@ -54,10 +50,12 @@ import org.apache.cassandra.index.*;
 import org.apache.cassandra.index.internal.composites.CompositesSearcher;
 import org.apache.cassandra.index.internal.keys.KeysSearcher;
 import org.apache.cassandra.index.transactions.IndexTransaction;
-import org.apache.cassandra.index.transactions.UpdateTransaction;
 import org.apache.cassandra.io.sstable.ReducingKeyIterator;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.schema.IndexMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.concurrent.Refs;
@@ -271,14 +269,6 @@ public abstract class CassandraIndex implements Index
         return indexCfs.getMeanRowCount();
     }
 
-    /**
-     * No post processing of query results, just return them unchanged
-     */
-    public BiFunction<PartitionIterator, ReadCommand, PartitionIterator> postProcessorFor(ReadCommand command)
-    {
-        return (partitionIterator, readCommand) -> partitionIterator;
-    }
-
     public RowFilter getPostIndexQueryFilter(RowFilter filter)
     {
         return getTargetExpression(filter.getExpressions()).map(filter::without)
@@ -340,7 +330,8 @@ public abstract class CassandraIndex implements Index
                               final RegularAndStaticColumns columns,
                               final int nowInSec,
                               final WriteContext ctx,
-                              final IndexTransaction.Type transactionType)
+                              final IndexTransaction.Type transactionType,
+                              Memtable memtable)
     {
         /**
          * Indexes on regular and static columns (the non primary-key ones) only care about updates with live
@@ -524,7 +515,7 @@ public abstract class CassandraIndex implements Index
                                                                cell));
         Row row = BTreeRow.noCellLiveRow(buildIndexClustering(rowKey, clustering, cell), info);
         PartitionUpdate upd = partitionUpdate(valueKey, row);
-        indexCfs.getWriteHandler().write(upd, ctx, UpdateTransaction.NO_OP);
+        indexCfs.getWriteHandler().write(upd, ctx, false);
         logger.trace("Inserted entry into index for value {}", valueKey);
     }
 
@@ -570,7 +561,7 @@ public abstract class CassandraIndex implements Index
     {
         Row row = BTreeRow.emptyDeletedRow(indexClustering, Row.Deletion.regular(deletion));
         PartitionUpdate upd = partitionUpdate(indexKey, row);
-        indexCfs.getWriteHandler().write(upd, ctx, UpdateTransaction.NO_OP);
+        indexCfs.getWriteHandler().write(upd, ctx, false);
         logger.trace("Removed index entry for value {}", indexKey);
     }
 
diff --git a/src/java/org/apache/cassandra/index/internal/CassandraIndexSearcher.java b/src/java/org/apache/cassandra/index/internal/CassandraIndexSearcher.java
index 0935005e0e..720de13c40 100644
--- a/src/java/org/apache/cassandra/index/internal/CassandraIndexSearcher.java
+++ b/src/java/org/apache/cassandra/index/internal/CassandraIndexSearcher.java
@@ -54,6 +54,12 @@ public abstract class CassandraIndexSearcher implements Index.Searcher
         this.index = index;
     }
 
+    @Override
+    public ReadCommand command()
+    {
+        return command;
+    }
+
     @SuppressWarnings("resource") // Both the OpOrder and 'indexIter' are closed on exception, or through the closing of the result
     // of this method.
     public UnfilteredPartitionIterator search(ReadExecutionController executionController)
diff --git a/src/java/org/apache/cassandra/index/sasi/SASIIndex.java b/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
index 38588d3283..7413520077 100644
--- a/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
+++ b/src/java/org/apache/cassandra/index/sasi/SASIIndex.java
@@ -27,7 +27,6 @@ import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.concurrent.Callable;
-import java.util.function.BiFunction;
 
 import com.googlecode.concurrenttrees.common.Iterables;
 import org.apache.cassandra.config.DatabaseDescriptor;
@@ -44,9 +43,10 @@ import org.apache.cassandra.db.WriteContext;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.lifecycle.Tracker;
 import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.dht.Murmur3Partitioner;
@@ -60,7 +60,7 @@ import org.apache.cassandra.index.sasi.conf.ColumnIndex;
 import org.apache.cassandra.index.sasi.conf.IndexMode;
 import org.apache.cassandra.index.sasi.disk.OnDiskIndexBuilder.Mode;
 import org.apache.cassandra.index.sasi.disk.PerSSTableIndexWriter;
-import org.apache.cassandra.index.sasi.plan.QueryPlan;
+import org.apache.cassandra.index.sasi.plan.SASIIndexSearcher;
 import org.apache.cassandra.index.transactions.IndexTransaction;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableFlushObserver;
@@ -90,7 +90,8 @@ public class SASIIndex implements Index, INotificationConsumer
     {
         public SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs,
                                                        Set<Index> indexes,
-                                                       Collection<SSTableReader> sstablesToRebuild)
+                                                       Collection<SSTableReader> sstablesToRebuild,
+                                                       boolean isFullRebuild)
         {
             NavigableMap<SSTableReader, Map<ColumnMetadata, ColumnIndex>> sstables = new TreeMap<>(SSTableReader.idComparator);
 
@@ -182,9 +183,10 @@ public class SASIIndex implements Index, INotificationConsumer
         return Collections.emptyMap();
     }
 
+    @Override
     public void register(IndexRegistry registry)
     {
-        registry.registerIndex(this);
+        registry.registerIndex(this, this, () -> new SASIIndexGroup(this));
     }
 
     public IndexMetadata getIndexMetadata()
@@ -267,12 +269,7 @@ public class SASIIndex implements Index, INotificationConsumer
     {}
 
     @Override
-    public boolean supportsReplicaFilteringProtection(RowFilter rowFilter)
-    {
-        return false;
-    }
-
-    public Indexer indexerFor(DecoratedKey key, RegularAndStaticColumns columns, int nowInSec, WriteContext context, IndexTransaction.Type transactionType)
+    public Indexer indexerFor(DecoratedKey key, RegularAndStaticColumns columns, int nowInSec, WriteContext context, IndexTransaction.Type transactionType, Memtable memtable)
     {
         return new Indexer()
         {
@@ -320,17 +317,12 @@ public class SASIIndex implements Index, INotificationConsumer
     {
         TableMetadata config = command.metadata();
         ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(config.id);
-        return controller -> new QueryPlan(cfs, command, DatabaseDescriptor.getRangeRpcTimeout(MILLISECONDS)).execute(controller);
-    }
-
-    public SSTableFlushObserver getFlushObserver(Descriptor descriptor, OperationType opType)
-    {
-        return newWriter(baseCfs.metadata().partitionKeyType, descriptor, Collections.singletonMap(index.getDefinition(), index), opType);
+        return new SASIIndexSearcher(cfs, command, DatabaseDescriptor.getRangeRpcTimeout(MILLISECONDS));
     }
 
-    public BiFunction<PartitionIterator, ReadCommand, PartitionIterator> postProcessorFor(ReadCommand command)
+    public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker)
     {
-        return (partitionIterator, readCommand) -> partitionIterator;
+        return newWriter(baseCfs.metadata().partitionKeyType, descriptor, Collections.singletonMap(index.getDefinition(), index), tracker.opType());
     }
 
     public IndexBuildingSupport getBuildTaskSupport()
diff --git a/src/java/org/apache/cassandra/db/TableWriteHandler.java b/src/java/org/apache/cassandra/index/sasi/SASIIndexGroup.java
similarity index 59%
copy from src/java/org/apache/cassandra/db/TableWriteHandler.java
copy to src/java/org/apache/cassandra/index/sasi/SASIIndexGroup.java
index 4e47221925..8b4c6fa0cb 100644
--- a/src/java/org/apache/cassandra/db/TableWriteHandler.java
+++ b/src/java/org/apache/cassandra/index/sasi/SASIIndexGroup.java
@@ -16,12 +16,23 @@
  * limitations under the License.
  */
 
-package org.apache.cassandra.db;
+package org.apache.cassandra.index.sasi;
 
-import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.index.transactions.UpdateTransaction;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.SingletonIndexGroup;
+import org.apache.cassandra.index.sasi.plan.SASIIndexQueryPlan;
 
-public interface TableWriteHandler
+public class SASIIndexGroup extends SingletonIndexGroup
 {
-    void write(PartitionUpdate update, WriteContext context, UpdateTransaction updateTransaction);
+    public SASIIndexGroup(SASIIndex index)
+    {
+        super(index);
+    }
+
+    @Override
+    public Index.QueryPlan queryPlanFor(RowFilter rowFilter)
+    {
+        return SASIIndexQueryPlan.create((SASIIndex) getIndex(), rowFilter);
+    }
 }
diff --git a/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java b/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java
index f89ff62e28..afd58896d3 100644
--- a/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java
+++ b/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java
@@ -282,7 +282,7 @@ public class PerSSTableIndexWriter implements SSTableFlushObserver
         {
             logger.info("Scheduling index flush to {}", outputFile);
 
-            getExecutor().submit((Runnable) () -> {
+            getExecutor().submit(() -> {
                 long start1 = nanoTime();
 
                 OnDiskIndex[] parts = new OnDiskIndex[segments.size() + 1];
@@ -299,7 +299,7 @@ public class PerSSTableIndexWriter implements SSTableFlushObserver
                     // parts are present but there is something still in memory, let's flush that inline
                     if (!currentBuilder.isEmpty())
                     {
-                        @SuppressWarnings("resource")
+                        @SuppressWarnings({ "resource", "RedundantSuppression" })
                         OnDiskIndex last = scheduleSegmentFlush(false).call();
                         segments.add(ImmediateFuture.success(last));
                     }
@@ -309,7 +309,7 @@ public class PerSSTableIndexWriter implements SSTableFlushObserver
 
                     for (Future<OnDiskIndex> f : segments)
                     {
-                        @SuppressWarnings("resource")
+                        @SuppressWarnings({ "resource", "RedundantSuppression" })
                         OnDiskIndex part = f.get();
                         if (part == null)
                             continue;
@@ -335,7 +335,7 @@ public class PerSSTableIndexWriter implements SSTableFlushObserver
 
                     for (int segment = 0; segment < segmentNumber; segment++)
                     {
-                        @SuppressWarnings("resource")
+                        @SuppressWarnings({ "resource", "RedundantSuppression" })
                         OnDiskIndex part = parts[segment];
 
                         if (part != null)
@@ -378,6 +378,6 @@ public class PerSSTableIndexWriter implements SSTableFlushObserver
 
     public boolean equals(Object o)
     {
-        return !(o == null || !(o instanceof PerSSTableIndexWriter)) && descriptor.equals(((PerSSTableIndexWriter) o).descriptor);
+        return o instanceof PerSSTableIndexWriter && descriptor.equals(((PerSSTableIndexWriter) o).descriptor);
     }
 }
diff --git a/src/java/org/apache/cassandra/index/sasi/plan/SASIIndexQueryPlan.java b/src/java/org/apache/cassandra/index/sasi/plan/SASIIndexQueryPlan.java
new file mode 100644
index 0000000000..06664a1834
--- /dev/null
+++ b/src/java/org/apache/cassandra/index/sasi/plan/SASIIndexQueryPlan.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index.sasi.plan;
+
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.index.SingletonIndexQueryPlan;
+import org.apache.cassandra.index.sasi.SASIIndex;
+
+public class SASIIndexQueryPlan extends SingletonIndexQueryPlan
+{
+    private SASIIndexQueryPlan(SASIIndex index, RowFilter postIndexFilter)
+    {
+        super(index, postIndexFilter);
+    }
+
+    @Nullable
+    public static SASIIndexQueryPlan create(SASIIndex index, RowFilter rowFilter)
+    {
+        for (RowFilter.Expression e : rowFilter.getExpressions())
+        {
+            if (index.supportsExpression(e.column(), e.operator()))
+                return new SASIIndexQueryPlan(index, index.getPostIndexQueryFilter(rowFilter));
+        }
+
+        return null;
+    }
+
+    @Override
+    public boolean supportsReplicaFilteringProtection(RowFilter rowFilter)
+    {
+        return false;
+    }
+}
diff --git a/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java b/src/java/org/apache/cassandra/index/sasi/plan/SASIIndexSearcher.java
similarity index 92%
rename from src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java
rename to src/java/org/apache/cassandra/index/sasi/plan/SASIIndexSearcher.java
index a54dfc8dd5..10d2aab5c4 100644
--- a/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java
+++ b/src/java/org/apache/cassandra/index/sasi/plan/SASIIndexSearcher.java
@@ -23,22 +23,30 @@ import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.db.rows.*;
 import org.apache.cassandra.dht.AbstractBounds;
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.sasi.disk.Token;
 import org.apache.cassandra.index.sasi.plan.Operation.OperationType;
-import org.apache.cassandra.exceptions.RequestTimeoutException;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.AbstractIterator;
 
-public class QueryPlan
+public class SASIIndexSearcher implements Index.Searcher
 {
+    private final ReadCommand command;
     private final QueryController controller;
 
-    public QueryPlan(ColumnFamilyStore cfs, ReadCommand command, long executionQuotaMs)
+    public SASIIndexSearcher(ColumnFamilyStore cfs, ReadCommand command, long executionQuotaMs)
     {
+        this.command = command;
         this.controller = new QueryController(cfs, (PartitionRangeReadCommand) command, executionQuotaMs);
     }
 
+    @Override
+    public ReadCommand command()
+    {
+        return command;
+    }
+
     /**
      * Converts expressions into operation tree (which is currently just a single AND).
      *
@@ -63,7 +71,8 @@ public class QueryPlan
         }
     }
 
-    public UnfilteredPartitionIterator execute(ReadExecutionController executionController) throws RequestTimeoutException
+    @Override
+    public UnfilteredPartitionIterator search(ReadExecutionController executionController)
     {
         return new ResultIterator(analyze(), controller, executionController);
     }
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTable.java b/src/java/org/apache/cassandra/io/sstable/SSTable.java
index 5156adc4e1..e9f10570ba 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTable.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTable.java
@@ -65,8 +65,6 @@ import static org.apache.cassandra.service.ActiveRepairService.UNREPAIRED_SSTABL
  */
 public abstract class SSTable
 {
-    static final Logger logger = LoggerFactory.getLogger(SSTable.class);
-
     public static final int TOMBSTONE_HISTOGRAM_BIN_SIZE = 100;
     public static final int TOMBSTONE_HISTOGRAM_SPOOL_SIZE = 100000;
     public static final int TOMBSTONE_HISTOGRAM_TTL_ROUND_SECONDS = CassandraRelevantProperties.STREAMING_HISTOGRAM_ROUND_SECONDS.getInt();
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableFlushObserver.java b/src/java/org/apache/cassandra/io/sstable/SSTableFlushObserver.java
index 025d8cbaf4..159e0d43bd 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableFlushObserver.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableFlushObserver.java
@@ -38,12 +38,12 @@ public interface SSTableFlushObserver
      *
      * @param key                the key being appended to SSTable.
      * @param keyPosition        the position of the key in the SSTable data file
-     * @param KeyPositionForSASI SSTable format specific key position for storage attached indexes, it can be
+     * @param keyPositionForSASI SSTable format specific key position for storage attached indexes, it can be
      *                           in data file or in some index file. It is the same position as returned by
      *                           {@link KeyReader#keyPositionForSecondaryIndex()} for the same format, and the same
      *                           position as expected by {@link SSTableReader#keyAtPositionFromSecondaryIndex(long)}.
      */
-    void startPartition(DecoratedKey key, long keyPosition, long KeyPositionForSASI);
+    void startPartition(DecoratedKey key, long keyPosition, long keyPositionForSASI);
 
     /**
      * Called when a static row is being written to the sstable. If static columns are present in the table, it is called
@@ -54,16 +54,23 @@ public interface SSTableFlushObserver
     void staticRow(Row staticRow);
 
     /**
-     * Called after the unfiltered cluster is written to the sstable.
-     * Will be preceded by a call to {@code startPartition(DecoratedKey, long)},
-     * and the cluster should be assumed to belong to that partition.
+     * Called after an unfiltered is written to the sstable.
      *
-     * @param unfilteredCluster The unfiltered cluster being added to SSTable.
+     * Will be preceded by a call to {@link #startPartition(DecoratedKey, long, long)},
+     * and the unfiltered should be assumed to belong to that partition.
+     *
+     * @param unfiltered the unfiltered being written to the SSTable
      */
-    void nextUnfilteredCluster(Unfiltered unfilteredCluster);
+    void nextUnfilteredCluster(Unfiltered unfiltered);
 
     /**
      * Called when all data is written to the file and it's ready to be finished up.
      */
     void complete();
+
+    /**
+     * Clean up resources on error. There should be no side effects if called multiple times.
+     */
+    @SuppressWarnings("unused")
+    default void abort(Throwable accumulator) {}
 }
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java
index fad7d54424..22099bec26 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java
@@ -97,7 +97,7 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem
         return writer.finished();
     }
 
-    @SuppressWarnings("resource") // log and writer closed during doPostCleanup
+    @SuppressWarnings({"resource", "RedundantSuppression"}) // log and writer closed during doPostCleanup
     public static SSTableTxnWriter create(ColumnFamilyStore cfs, Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, int sstableLevel, SerializationHeader header)
     {
         LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
@@ -106,7 +106,7 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem
     }
 
 
-    @SuppressWarnings("resource") // log and writer closed during doPostCleanup
+    @SuppressWarnings({"resource", "RedundantSuppression"}) // log and writer closed during doPostCleanup
     public static SSTableTxnWriter createRangeAware(TableMetadataRef metadata,
                                                     long keyCount,
                                                     long repairedAt,
@@ -134,7 +134,7 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem
         return new SSTableTxnWriter(txn, writer);
     }
 
-    @SuppressWarnings("resource") // log and writer closed during doPostCleanup
+    @SuppressWarnings({"resource", "RedundantSuppression"}) // log and writer closed during doPostCleanup
     public static SSTableTxnWriter create(TableMetadataRef metadata,
                                           Descriptor descriptor,
                                           long keyCount,
@@ -143,13 +143,13 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem
                                           boolean isTransient,
                                           int sstableLevel,
                                           SerializationHeader header,
-                                          Collection<Index> indexes,
+                                          Collection<Index.Group> indexGroups,
                                           SSTable.Owner owner)
     {
         // if the column family store does not exist, we create a new default SSTableMultiWriter to use:
         LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
         MetadataCollector collector = new MetadataCollector(metadata.get().comparator).sstableLevel(sstableLevel);
-        SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, collector, header, indexes, txn, owner);
+        SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, collector, header, indexGroups, txn, owner);
         return new SSTableTxnWriter(txn, writer);
     }
 
diff --git a/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java b/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java
index 381c5eb33f..c93a2368e1 100644
--- a/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java
@@ -107,7 +107,7 @@ public class SimpleSSTableMultiWriter implements SSTableMultiWriter
         writer.close();
     }
 
-    @SuppressWarnings("resource") // SimpleSSTableMultiWriter closes writer
+    @SuppressWarnings({"resource", "RedundantSuppression"}) // SimpleSSTableMultiWriter closes writer
     public static SSTableMultiWriter create(Descriptor descriptor,
                                             long keyCount,
                                             long repairedAt,
@@ -116,7 +116,7 @@ public class SimpleSSTableMultiWriter implements SSTableMultiWriter
                                             TableMetadataRef metadata,
                                             MetadataCollector metadataCollector,
                                             SerializationHeader header,
-                                            Collection<Index> indexes,
+                                            Collection<Index.Group> indexGroups,
                                             LifecycleNewTracker lifecycleNewTracker,
                                             SSTable.Owner owner)
     {
@@ -128,8 +128,8 @@ public class SimpleSSTableMultiWriter implements SSTableMultiWriter
                                             .setTableMetadataRef(metadata)
                                             .setMetadataCollector(metadataCollector)
                                             .setSerializationHeader(header)
-                                            .addDefaultComponents()
-                                            .addFlushObserversForSecondaryIndexes(indexes, lifecycleNewTracker.opType())
+                                            .addDefaultComponents(indexGroups)
+                                            .addFlushObserversForSecondaryIndexes(indexGroups, lifecycleNewTracker, metadata.get())
                                             .build(lifecycleNewTracker, owner);
         return new SimpleSSTableMultiWriter(writer, lifecycleNewTracker);
     }
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
index 255e1cd04a..71380d7a7b 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
@@ -21,8 +21,10 @@ package org.apache.cassandra.io.sstable.format;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
 
@@ -31,7 +33,6 @@ import com.google.common.collect.ImmutableSet;
 
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.dht.AbstractBounds;
@@ -39,6 +40,7 @@ import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
+import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.SSTableFlushObserver;
@@ -49,6 +51,7 @@ import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
 import org.apache.cassandra.io.sstable.metadata.MetadataType;
 import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
 import org.apache.cassandra.io.util.MmappedRegionsCache;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Transactional;
@@ -233,7 +236,14 @@ public abstract class SSTableWriter extends SSTable implements Transactional
 
     public final Throwable abort(Throwable accumulate)
     {
-        return txnProxy.abort(accumulate);
+        try
+        {
+            return txnProxy.abort(accumulate);
+        }
+        finally
+        {
+            observers.forEach(observer -> observer.abort(accumulate));
+        }
     }
 
     public final void close()
@@ -243,7 +253,14 @@ public abstract class SSTableWriter extends SSTable implements Transactional
 
     public final void abort()
     {
-        txnProxy.abort();
+        try
+        {
+            txnProxy.abort();
+        }
+        finally
+        {
+            observers.forEach(observer -> observer.abort(null));
+        }
     }
 
     protected Map<MetadataType, MetadataComponent> finalizeMetadata()
@@ -400,7 +417,7 @@ public abstract class SSTableWriter extends SSTable implements Transactional
             return (B) this;
         }
 
-        public B addDefaultComponents()
+        public B addDefaultComponents(Collection<Index.Group> indexGroups)
         {
             checkNotNull(getTableMetadataRef());
 
@@ -417,21 +434,35 @@ public abstract class SSTableWriter extends SSTable implements Transactional
                 addComponents(ImmutableSet.of(Components.CRC));
             }
 
+            if (!indexGroups.isEmpty())
+                addComponents(indexComponents(indexGroups));
+
             return (B) this;
         }
 
-        public B addFlushObserversForSecondaryIndexes(Collection<Index> indexes, OperationType operationType)
+        private static Set<Component> indexComponents(Collection<Index.Group> indexGroups)
+        {
+            Set<Component> components = new HashSet<>();
+            for (Index.Group group : indexGroups)
+            {
+                components.addAll(group.getComponents());
+            }
+
+            return components;
+        }
+
+        public B addFlushObserversForSecondaryIndexes(Collection<Index.Group> indexGroups, LifecycleNewTracker tracker, TableMetadata metadata)
         {
-            if (indexes == null)
+            if (indexGroups == null)
                 return (B) this;
 
             Collection<SSTableFlushObserver> current = this.flushObservers != null ? this.flushObservers : Collections.emptyList();
-            List<SSTableFlushObserver> observers = new ArrayList<>(indexes.size() + current.size());
+            List<SSTableFlushObserver> observers = new ArrayList<>(indexGroups.size() + current.size());
             observers.addAll(current);
 
-            for (Index index : indexes)
+            for (Index.Group group : indexGroups)
             {
-                SSTableFlushObserver observer = index.getFlushObserver(descriptor, operationType);
+                SSTableFlushObserver observer = group.getFlushObserver(descriptor, tracker, metadata);
                 if (observer != null)
                 {
                     observer.begin();
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTableWriter.java
index 2deda2a6f7..455e3c6a83 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SortedTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTableWriter.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.io.sstable.format;
 
 import java.io.IOException;
 import java.nio.BufferOverflowException;
+import java.util.Collection;
 import java.util.Set;
 import java.util.function.Consumer;
 
@@ -43,6 +44,7 @@ import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.Rows;
 import org.apache.cassandra.db.rows.Unfiltered;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.io.compress.CompressedSequentialWriter;
 import org.apache.cassandra.io.compress.CompressionMetadata;
@@ -459,9 +461,9 @@ public abstract class SortedTableWriter<P extends SortedTablePartitionWriter> ex
         }
 
         @Override
-        public B addDefaultComponents()
+        public B addDefaultComponents(Collection<Index.Group> indexGroups)
         {
-            super.addDefaultComponents();
+            super.addDefaultComponents(indexGroups);
 
             if (FilterComponent.shouldUseBloomFilter(getTableMetadataRef().getLocal().params.bloomFilterFpChance))
             {
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
index 88ca15c527..97f244ca66 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
@@ -39,11 +39,11 @@ import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.GaugeProvider;
 import org.apache.cassandra.io.sstable.IScrubber;
 import org.apache.cassandra.io.sstable.MetricsProviders;
-import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.filter.BloomFilterMetrics;
 import org.apache.cassandra.io.sstable.format.AbstractSSTableFormat;
 import org.apache.cassandra.io.sstable.format.SSTableFormat;
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
index 5ae52cfb6c..1a49573e19 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
@@ -19,6 +19,7 @@ package org.apache.cassandra.io.sstable.format.big;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Consumer;
@@ -33,6 +34,7 @@ import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.DeletionTime;
 import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
 import org.apache.cassandra.io.sstable.Descriptor;
@@ -141,7 +143,7 @@ public class BigTableWriter extends SortedTableWriter<BigFormatPartitionWriter>
         return entry;
     }
 
-    @SuppressWarnings("resource")
+    @SuppressWarnings({"resource", "RedundantSuppression"})
     private BigTableReader openInternal(IndexSummaryBuilder.ReadableBoundary boundary, SSTableReader.OpenReason openReason)
     {
         assert boundary == null || (boundary.indexLength > 0 && boundary.dataLength > 0);
@@ -276,7 +278,7 @@ public class BigTableWriter extends SortedTableWriter<BigFormatPartitionWriter>
             summary = new IndexSummaryBuilder(b.getKeyCount(), b.getTableMetadataRef().getLocal().params.minIndexInterval, Downsampling.BASE_SAMPLING_LEVEL);
             // register listeners to be alerted when the data files are flushed
             writer.setPostFlushListener(() -> summary.markIndexSynced(writer.getLastFlushOffset()));
-            @SuppressWarnings("resource")
+            @SuppressWarnings({"resource", "RedundantSuppression"})
             SequentialWriter dataWriter = b.getDataWriter();
             dataWriter.setPostFlushListener(() -> summary.markDataSynced(dataWriter.getLastFlushOffset()));
         }
@@ -382,9 +384,9 @@ public class BigTableWriter extends SortedTableWriter<BigFormatPartitionWriter>
         }
 
         @Override
-        public Builder addDefaultComponents()
+        public Builder addDefaultComponents(Collection<Index.Group> indexGroups)
         {
-            super.addDefaultComponents();
+            super.addDefaultComponents(indexGroups);
 
             addComponents(ImmutableSet.of(Components.PRIMARY_INDEX, Components.SUMMARY));
 
diff --git a/src/java/org/apache/cassandra/metrics/ClientRangeRequestMetrics.java b/src/java/org/apache/cassandra/metrics/ClientRangeRequestMetrics.java
new file mode 100644
index 0000000000..c974651381
--- /dev/null
+++ b/src/java/org/apache/cassandra/metrics/ClientRangeRequestMetrics.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cassandra.metrics;
+
+import com.codahale.metrics.Histogram;
+
+import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
+
+/**
+ * Metrics for tracking information about range requests.
+ *
+ */
+public class ClientRangeRequestMetrics extends ClientRequestMetrics
+{
+    /**
+     * Metric for tracking the number of round trips made for a range request.
+     */
+    public final Histogram roundTrips;
+
+    public ClientRangeRequestMetrics(String scope)
+    {
+        super(scope);
+        roundTrips = Metrics.histogram(factory.createMetricName("RoundTripsPerReadHistogram"), false);
+    }
+
+    public void release()
+    {
+        super.release();
+        Metrics.remove(factory.createMetricName("RoundTripsPerReadHistogram"));
+    }
+}
diff --git a/src/java/org/apache/cassandra/schema/IndexMetadata.java b/src/java/org/apache/cassandra/schema/IndexMetadata.java
index 4adbce170b..cdba139b5d 100644
--- a/src/java/org/apache/cassandra/schema/IndexMetadata.java
+++ b/src/java/org/apache/cassandra/schema/IndexMetadata.java
@@ -21,6 +21,7 @@ package org.apache.cassandra.schema;
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
@@ -37,6 +38,7 @@ import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.exceptions.UnknownIndexException;
 import org.apache.cassandra.index.Index;
+import org.apache.cassandra.index.sasi.SASIIndex;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.utils.FBUtilities;
@@ -55,6 +57,16 @@ public final class IndexMetadata
 
     public static final Serializer serializer = new Serializer();
 
+    /**
+     * A mapping of user-friendly index names to their fully qualified index class names.
+     */
+    private static final Map<String, String> indexNameAliases = new ConcurrentHashMap<>();
+
+    static
+    {
+        indexNameAliases.put(SASIIndex.class.getSimpleName(), SASIIndex.class.getCanonicalName());
+    }
+
     public enum Kind
     {
         KEYS, CUSTOM, COMPOSITES
@@ -122,7 +134,10 @@ public final class IndexMetadata
             if (options == null || !options.containsKey(IndexTarget.CUSTOM_INDEX_OPTION_NAME))
                 throw new ConfigurationException(String.format("Required option missing for index %s : %s",
                                                                name, IndexTarget.CUSTOM_INDEX_OPTION_NAME));
-            String className = options.get(IndexTarget.CUSTOM_INDEX_OPTION_NAME);
+
+            // Find any aliases to the fully qualified index class name:
+            String className = expandAliases(options.get(IndexTarget.CUSTOM_INDEX_OPTION_NAME));
+
             Class<Index> indexerClass = FBUtilities.classForName(className, "custom indexer");
             if (!Index.class.isAssignableFrom(indexerClass))
                 throw new ConfigurationException(String.format("Specified Indexer class (%s) does not implement the Indexer interface", className));
@@ -130,6 +145,11 @@ public final class IndexMetadata
         }
     }
 
+    public static String expandAliases(String className)
+    {
+        return indexNameAliases.getOrDefault(className, className);
+    }
+
     private void validateCustomIndexOptions(TableMetadata table, Class<? extends Index> indexerClass, Map<String, String> options)
     {
         try
diff --git a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java
index b03701d10b..a27a6dc196 100644
--- a/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java
+++ b/src/java/org/apache/cassandra/service/paxos/uncommitted/PaxosUncommittedIndex.java
@@ -245,7 +245,7 @@ public class PaxosUncommittedIndex implements Index, PaxosUncommittedTracker.Upd
 
     }
 
-    public Indexer indexerFor(DecoratedKey key, RegularAndStaticColumns columns, int nowInSec, WriteContext ctx, IndexTransaction.Type transactionType)
+    public Indexer indexerFor(DecoratedKey key, RegularAndStaticColumns columns, int nowInSec, WriteContext ctx, IndexTransaction.Type transactionType, Memtable memtable)
     {
         return indexer;
     }
diff --git a/src/java/org/apache/cassandra/service/reads/DataResolver.java b/src/java/org/apache/cassandra/service/reads/DataResolver.java
index e7e49521c1..66719554b9 100644
--- a/src/java/org/apache/cassandra/service/reads/DataResolver.java
+++ b/src/java/org/apache/cassandra/service/reads/DataResolver.java
@@ -29,7 +29,6 @@ import javax.annotation.Nullable;
 import com.google.common.base.Joiner;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
-import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.DeletionTime;
 import org.apache.cassandra.db.ReadCommand;
@@ -51,7 +50,6 @@ import org.apache.cassandra.index.Index;
 import org.apache.cassandra.locator.Endpoints;
 import org.apache.cassandra.locator.ReplicaPlan;
 import org.apache.cassandra.net.Message;
-import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.service.reads.repair.NoopReadRepair;
 import org.apache.cassandra.service.reads.repair.ReadRepair;
@@ -137,22 +135,11 @@ public class DataResolver<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<
         if (command.rowFilter().isEmpty())
             return false;
 
-        IndexMetadata indexMetadata = command.indexMetadata();
-
-        if (indexMetadata == null || !indexMetadata.isCustom())
-        {
+        Index.QueryPlan queryPlan = command.indexQueryPlan();
+        if (queryPlan == null )
             return true;
-        }
-
-        ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(command.metadata().id);
-
-        assert cfs != null;
-
-        Index index = command.getIndex(cfs);
 
-        assert index != null;
-
-        return index.supportsReplicaFilteringProtection(command.rowFilter());
+        return queryPlan.supportsReplicaFilteringProtection(command.rowFilter());
     }
 
     private class ResolveContext
@@ -258,13 +245,24 @@ public class DataResolver<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<
 
         PartitionIterator completedPartitions = resolveWithReadRepair(secondPhaseContext,
                                                                       i -> rfp.queryProtectedPartitions(firstPhasePartitions, i),
-                                                                      results -> command.rowFilter().filter(results, command.metadata(), command.nowInSec()),
+                                                                      preCountFilterForReplicaFilteringProtection(),
                                                                       repairedDataTracker);
 
         // Ensure that the RFP instance has a chance to record metrics when the iterator closes.
         return PartitionIterators.doOnClose(completedPartitions, firstPhasePartitions::close);
     }
 
+    private  UnaryOperator<PartitionIterator> preCountFilterForReplicaFilteringProtection()
+    {
+        return results -> {
+            Index.Searcher searcher = command.indexSearcher();
+            // in case of "ALLOW FILTERING" without index
+            if (searcher == null)
+                return command.rowFilter().filter(results, command.metadata(), command.nowInSec());
+            return searcher.filterReplicaFilteringProtection(results);
+        };
+    }
+
     @SuppressWarnings("resource")
     private PartitionIterator resolveInternal(ResolveContext context,
                                               UnfilteredPartitionIterators.MergeListener mergeListener,
diff --git a/src/java/org/apache/cassandra/service/reads/ShortReadRowsProtection.java b/src/java/org/apache/cassandra/service/reads/ShortReadRowsProtection.java
index 8061f0a008..4ed9e32956 100644
--- a/src/java/org/apache/cassandra/service/reads/ShortReadRowsProtection.java
+++ b/src/java/org/apache/cassandra/service/reads/ShortReadRowsProtection.java
@@ -184,6 +184,6 @@ class ShortReadRowsProtection extends Transformation implements MoreRows<Unfilte
                                                  command.limits().forShortReadRetry(toQuery),
                                                  partitionKey,
                                                  filter,
-                                                 command.indexMetadata());
+                                                 command.indexQueryPlan());
     }
 }
diff --git a/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java b/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java
index b353b4bbad..8881308534 100644
--- a/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java
+++ b/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java
@@ -42,7 +42,7 @@ import org.apache.cassandra.exceptions.UnavailableException;
 import org.apache.cassandra.locator.EndpointsForRange;
 import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.locator.ReplicaPlan;
-import org.apache.cassandra.metrics.ClientRequestMetrics;
+import org.apache.cassandra.metrics.ClientRangeRequestMetrics;
 import org.apache.cassandra.net.Message;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.service.StorageProxy;
@@ -60,7 +60,7 @@ public class RangeCommandIterator extends AbstractIterator<RowIterator> implemen
 {
     private static final Logger logger = LoggerFactory.getLogger(RangeCommandIterator.class);
 
-    private static final ClientRequestMetrics rangeMetrics = new ClientRequestMetrics("RangeSlice");
+    public static final ClientRangeRequestMetrics rangeMetrics = new ClientRangeRequestMetrics("RangeSlice");
 
     private final CloseableIterator<ReplicaPlan.ForRangeRead> replicaPlans;
     private final int totalRangeCount;
@@ -270,6 +270,7 @@ public class RangeCommandIterator extends AbstractIterator<RowIterator> implemen
         {
             long latency = nanoTime() - startTime;
             rangeMetrics.addNano(latency);
+            rangeMetrics.roundTrips.update(batchesRequested);
             Keyspace.openAndGetStore(command.metadata()).metric.coordinatorScanLatency.update(latency, TimeUnit.NANOSECONDS);
         }
     }
diff --git a/src/java/org/apache/cassandra/service/reads/range/RangeCommands.java b/src/java/org/apache/cassandra/service/reads/range/RangeCommands.java
index 1c5ce7f147..ca3c220d70 100644
--- a/src/java/org/apache/cassandra/service/reads/range/RangeCommands.java
+++ b/src/java/org/apache/cassandra/service/reads/range/RangeCommands.java
@@ -109,7 +109,7 @@ public class RangeCommands
     static float estimateResultsPerRange(PartitionRangeReadCommand command, Keyspace keyspace)
     {
         ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.metadata().id);
-        Index index = command.getIndex(cfs);
+        Index.QueryPlan index = command.indexQueryPlan();
         float maxExpectedResults = index == null
                                    ? command.limits().estimateTotalResults(cfs)
                                    : index.getEstimatedResultRows();
diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeIndexTestBase.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeIndexTestBase.java
new file mode 100644
index 0000000000..923f29abd4
--- /dev/null
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeIndexTestBase.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.distributed.upgrade;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import com.vdurmont.semver4j.Semver;
+import org.apache.cassandra.distributed.UpgradeableCluster;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.IUpgradeableInstance;
+
+import static java.lang.String.format;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL;
+import static org.apache.cassandra.distributed.api.ConsistencyLevel.ONE;
+import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM;
+import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
+import static org.apache.cassandra.distributed.shared.AssertUtils.row;
+
+public class MixedModeIndexTestBase extends UpgradeTestBase
+{
+    private static final AtomicInteger TABLE_INDEX = new AtomicInteger(1);
+
+    protected static void testIndex(Semver initial) throws Throwable
+    {
+        List<Tester> testers = new ArrayList<>();
+        testers.addAll(Tester.create(1, ALL));
+        testers.addAll(Tester.create(2, ALL, QUORUM));
+        testers.addAll(Tester.create(3, ALL, QUORUM, ONE));
+
+        new TestCase()
+        .nodes(3)
+        .nodesToUpgrade(1)
+        .upgradesToCurrentFrom(initial)
+        .withConfig(config -> config.set("read_request_timeout_in_ms", SECONDS.toMillis(30))
+                                    .set("write_request_timeout_in_ms", SECONDS.toMillis(30)))
+        .setup(cluster -> {
+            for (Tester tester : testers)
+            {
+                tester.createTable(cluster);
+                tester.createIndex(cluster);
+                tester.writeRows(cluster);
+            }
+        }).runAfterNodeUpgrade((cluster, node) -> {
+            for (Tester tester : testers)
+                tester.readRows(cluster);
+        }).run();
+    }
+
+    private static class Tester
+    {
+        private final String table;
+        private final int numWrittenReplicas;
+        private final ConsistencyLevel readConsistencyLevel;
+
+        private Tester(int numWrittenReplicas, ConsistencyLevel readConsistencyLevel)
+        {
+            this.table = "cf_" + TABLE_INDEX.getAndIncrement();
+            this.numWrittenReplicas = numWrittenReplicas;
+            this.readConsistencyLevel = readConsistencyLevel;
+        }
+
+        private static List<Tester> create(int numWrittenReplicas, ConsistencyLevel... readConsistencyLevels)
+        {
+            return Stream.of(readConsistencyLevels)
+                         .map(readConsistencyLevel -> new Tester(numWrittenReplicas, readConsistencyLevel))
+                         .collect(Collectors.toList());
+        }
+
+        private void createTable(UpgradeableCluster cluster)
+        {
+            cluster.schemaChange(withKeyspace(String.format("CREATE TABLE %%s.%s (k int, c int, v int, PRIMARY KEY (k, c))", table)));
+        }
+
+        public void createIndex(UpgradeableCluster cluster)
+        {
+            cluster.schemaChange(withKeyspace(String.format("CREATE INDEX ON %%s.%s (v);", table)));
+        }
+
+        private void writeRows(UpgradeableCluster cluster)
+        {
+            String query = withKeyspace(String.format("INSERT INTO %%s.%s (k, c, v) VALUES (?, ?, ?)", table));
+            for (int i = 1; i <= numWrittenReplicas; i++)
+            {
+                IUpgradeableInstance node = cluster.get(i);
+                node.executeInternal(query, 1, 1, 10);
+                node.executeInternal(query, 1, 2, 20);
+                node.executeInternal(query, 1, 3, 30);
+            }
+        }
+
+        private void readRows(UpgradeableCluster cluster)
+        {
+            String query = withKeyspace(String.format("SELECT * FROM %%s.%s WHERE v = ? ALLOW FILTERING", table));
+            int coordinator = 1;
+            try
+            {
+                for (coordinator = 1; coordinator <= cluster.size(); coordinator++)
+                {
+                    assertRows(cluster.coordinator(coordinator).execute(query, readConsistencyLevel, 10),
+                               row(1, 1, 10));
+                    assertRows(cluster.coordinator(coordinator).execute(query, readConsistencyLevel, 20),
+                               row(1, 2, 20));
+                    assertRows(cluster.coordinator(coordinator).execute(query, readConsistencyLevel, 30),
+                               row(1, 3, 30));
+                }
+            }
+            catch (Throwable t)
+            {
+                String format = "Unexpected error reading rows with %d written replicas, CL=%s and coordinator=%s";
+                throw new AssertionError(format(format, numWrittenReplicas, readConsistencyLevel, coordinator), t);
+            }
+        }
+    }
+}
diff --git a/src/java/org/apache/cassandra/db/TableWriteHandler.java b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeIndexV40Test.java
similarity index 73%
copy from src/java/org/apache/cassandra/db/TableWriteHandler.java
copy to test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeIndexV40Test.java
index 4e47221925..bd4488b928 100644
--- a/src/java/org/apache/cassandra/db/TableWriteHandler.java
+++ b/test/distributed/org/apache/cassandra/distributed/upgrade/MixedModeIndexV40Test.java
@@ -16,12 +16,18 @@
  * limitations under the License.
  */
 
-package org.apache.cassandra.db;
+package org.apache.cassandra.distributed.upgrade;
 
-import org.apache.cassandra.db.partitions.PartitionUpdate;
-import org.apache.cassandra.index.transactions.UpdateTransaction;
+import org.junit.Test;
 
-public interface TableWriteHandler
+/**
+ * {@link MixedModeIndexTestBase} for upgrades from 4.0.
+ */
+public class MixedModeIndexV40Test extends MixedModeIndexTestBase
 {
-    void write(PartitionUpdate update, WriteContext context, UpdateTransaction updateTransaction);
+    @Test
+    public void testIndex() throws Throwable
+    {
+        testIndex(v40);
+    }
 }
diff --git a/test/unit/org/apache/cassandra/db/SecondaryIndexTest.java b/test/unit/org/apache/cassandra/db/SecondaryIndexTest.java
index 14137f13dc..c13ef60018 100644
--- a/test/unit/org/apache/cassandra/db/SecondaryIndexTest.java
+++ b/test/unit/org/apache/cassandra/db/SecondaryIndexTest.java
@@ -120,7 +120,7 @@ public class SecondaryIndexTest
                                       .filterOn("birthdate", Operator.EQ, 1L)
                                       .build();
 
-        Index.Searcher searcher = rc.getIndex(cfs).searcherFor(rc);
+        Index.Searcher searcher = rc.indexSearcher();
         try (ReadExecutionController executionController = rc.executionController();
              UnfilteredPartitionIterator pi = searcher.search(executionController))
         {
@@ -207,7 +207,7 @@ public class SecondaryIndexTest
 
         // verify that it's not being indexed under any other value either
         ReadCommand rc = Util.cmd(cfs).build();
-        assertNull(rc.getIndex(cfs));
+        assertNull(rc.indexSearcher());
 
         // resurrect w/ a newer timestamp
         new RowUpdateBuilder(cfs.metadata(), 2, "k1").clustering("c").add("birthdate", 1L).build().apply();;
@@ -225,13 +225,13 @@ public class SecondaryIndexTest
         // todo - checking the # of index searchers for the command is probably not the best thing to test here
         RowUpdateBuilder.deleteRow(cfs.metadata(), 3, "k1", "c").applyUnsafe();
         rc = Util.cmd(cfs).build();
-        assertNull(rc.getIndex(cfs));
+        assertNull(rc.indexSearcher());
 
         // make sure obsolete mutations don't generate an index entry
         // todo - checking the # of index searchers for the command is probably not the best thing to test here
         new RowUpdateBuilder(cfs.metadata(), 3, "k1").clustering("c").add("birthdate", 1L).build().apply();;
         rc = Util.cmd(cfs).build();
-        assertNull(rc.getIndex(cfs));
+        assertNull(rc.indexSearcher());
     }
 
     @Test
@@ -546,7 +546,7 @@ public class SecondaryIndexTest
                              .filterOn("notbirthdate", Operator.EQ, 0L)
                              .build();
 
-        assertEquals("notbirthdate_key_index", rc.indexMetadata().name);
+        assertEquals("notbirthdate_key_index", rc.indexQueryPlan().getFirst().getIndexMetadata().name);
     }
 
     private void assertIndexedNone(ColumnFamilyStore cfs, ByteBuffer col, Object val)
@@ -562,7 +562,7 @@ public class SecondaryIndexTest
         ColumnMetadata cdef = cfs.metadata().getColumn(col);
 
         ReadCommand rc = Util.cmd(cfs).filterOn(cdef.name.toString(), Operator.EQ, ((AbstractType) cdef.cellValueType()).decompose(val)).build();
-        Index.Searcher searcher = rc.getIndex(cfs).searcherFor(rc);
+        Index.Searcher searcher = rc.indexSearcher();
         if (count != 0)
             assertNotNull(searcher);
 
diff --git a/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java b/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java
index d77e1fefc0..2b8db525f1 100644
--- a/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java
+++ b/test/unit/org/apache/cassandra/db/SerializationHeaderTest.java
@@ -56,7 +56,7 @@ import org.apache.cassandra.schema.TableMetadataRef;
 
 public class SerializationHeaderTest
 {
-    private static String KEYSPACE = "SerializationHeaderTest";
+    private static final String KEYSPACE = "SerializationHeaderTest";
 
     static
     {
@@ -102,7 +102,7 @@ public class SerializationHeaderTest
                                                              .setSerializationHeader(header)
                                                              .setFlushObservers(Collections.emptyList())
                                                              .setMetadataCollector(new MetadataCollector(schema.comparator))
-                                                             .addDefaultComponents()
+                                                             .addDefaultComponents(Collections.emptySet())
                                                              .build(txn, null))
                 {
                     ColumnMetadata cd = schema.getColumn(v);
@@ -139,7 +139,7 @@ public class SerializationHeaderTest
                 {
                     UnfilteredRowIterator partition = partitions.next();
                     long value = Int32Type.instance.compose(((Row)partition.next()).getCell(columnRegular).buffer());
-                    Assert.assertEquals(value, (long)i);
+                    Assert.assertEquals(value, i);
                     Assert.assertTrue(partition.staticRow().isEmpty());
                     Assert.assertFalse(partition.hasNext());
                 }
diff --git a/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java
index 78497965d7..cb6d5c8241 100644
--- a/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/ActiveCompactionsTest.java
@@ -82,7 +82,7 @@ public class ActiveCompactionsTest extends CQLTester
         for (int ii = 0; ii < loopCount; ii++)
         {
             CountDownLatch trigger = new CountDownLatch(1);
-            SecondaryIndexBuilder builder = idx.getBuildTaskSupport().getIndexBuildTask(getCurrentColumnFamilyStore(), Collections.singleton(idx), sstables);
+            SecondaryIndexBuilder builder = idx.getBuildTaskSupport().getIndexBuildTask(getCurrentColumnFamilyStore(), Collections.singleton(idx), sstables, true);
             Future<?> f1 = es.submit(() -> {
                 Uninterruptibles.awaitUninterruptibly(trigger);
                 try
@@ -119,7 +119,7 @@ public class ActiveCompactionsTest extends CQLTester
 
         Index idx = getCurrentColumnFamilyStore().indexManager.getIndexByName(idxName);
         Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
-        SecondaryIndexBuilder builder = idx.getBuildTaskSupport().getIndexBuildTask(getCurrentColumnFamilyStore(), Collections.singleton(idx), sstables);
+        SecondaryIndexBuilder builder = idx.getBuildTaskSupport().getIndexBuildTask(getCurrentColumnFamilyStore(), Collections.singleton(idx), sstables, false);
 
         MockActiveCompactions mockActiveCompactions = new MockActiveCompactions();
         CompactionManager.instance.submitIndexBuild(builder, mockActiveCompactions).get();
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java b/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java
index 85c8ae092f..c2146fcd16 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java
@@ -163,9 +163,9 @@ public class RealTransactionsTest extends SchemaLoader
                 rewriter.switchWriter(desc.getFormat().getWriterFactory().builder(desc)
                                           .setTableMetadataRef(metadata)
                                           .setSerializationHeader(SerializationHeader.make(cfs.metadata(), txn.originals()))
-                                          .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexes(), txn.opType())
+                                          .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexGroups(), txn, metadata.get())
                                           .setMetadataCollector(new MetadataCollector(cfs.metadata().comparator))
-                                          .addDefaultComponents()
+                                          .addDefaultComponents(cfs.indexManager.listIndexGroups())
                                           .build(txn, cfs));
                 while (ci.hasNext())
                 {
diff --git a/test/unit/org/apache/cassandra/index/CustomIndexTest.java b/test/unit/org/apache/cassandra/index/CustomIndexTest.java
index 0d7fc9f59e..2a4486f7d4 100644
--- a/test/unit/org/apache/cassandra/index/CustomIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/CustomIndexTest.java
@@ -24,31 +24,42 @@ import java.util.*;
 import java.util.concurrent.Callable;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import org.junit.Test;
 
 import com.datastax.driver.core.exceptions.QueryValidationException;
 import org.apache.cassandra.Util;
-import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.cql3.ColumnIdentifier;
 import org.apache.cassandra.cql3.restrictions.IndexRestrictions;
 import org.apache.cassandra.cql3.restrictions.StatementRestrictions;
 import org.apache.cassandra.cql3.statements.schema.IndexTarget;
 import org.apache.cassandra.cql3.statements.ModificationStatement;
+import org.apache.cassandra.db.ColumnFamilyStore.FlushReason;
 import org.apache.cassandra.db.*;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
 import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableFlushObserver;
 import org.apache.cassandra.index.transactions.IndexTransaction;
 import org.apache.cassandra.schema.IndexMetadata;
 import org.apache.cassandra.schema.Indexes;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.transport.ProtocolVersion;
 import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.FBUtilities;
@@ -227,7 +238,7 @@ public class CustomIndexTest extends CQLTester
     }
 
     @Test
-    public void defaultIndexNameContainsTargetColumns() throws Throwable
+    public void defaultIndexNameContainsTargetColumns()
     {
         createTable("CREATE TABLE %s(k int, c int, v1 int, v2 int, PRIMARY KEY(k,c))");
 
@@ -253,7 +264,7 @@ public class CustomIndexTest extends CQLTester
     }
 
     @Test
-    public void createMultiColumnIndexes() throws Throwable
+    public void createMultiColumnIndexes()
     {
         // smoke test for various permutations of multicolumn indexes
         createTable("CREATE TABLE %s (" +
@@ -326,7 +337,7 @@ public class CustomIndexTest extends CQLTester
     }
 
     @Test
-    public void createMultiColumnIndexIncludingUserTypeColumn() throws Throwable
+    public void createMultiColumnIndexIncludingUserTypeColumn()
     {
         String myType = KEYSPACE + '.' + createType("CREATE TYPE %s (a int, b int)");
         createTable("CREATE TABLE %s (k int PRIMARY KEY, v1 int, v2 frozen<" + myType + ">)");
@@ -535,7 +546,7 @@ public class CustomIndexTest extends CQLTester
     }
 
     @Test
-    public void reloadIndexMetadataOnBaseCfsReload() throws Throwable
+    public void reloadIndexMetadataOnBaseCfsReload()
     {
         // verify that whenever the base table TableMetadata is reloaded, a reload of the index
         // metadata is performed
@@ -607,7 +618,7 @@ public class CustomIndexTest extends CQLTester
     }
 
     @Test
-    public void validateOptions() throws Throwable
+    public void validateOptions()
     {
         createTable("CREATE TABLE %s(k int, c int, v1 int, v2 int, PRIMARY KEY(k,c))");
         createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c, v2) USING '%s' WITH OPTIONS = {'foo':'bar'}",
@@ -617,7 +628,7 @@ public class CustomIndexTest extends CQLTester
     }
 
     @Test
-    public void validateOptionsWithTableMetadata() throws Throwable
+    public void validateOptionsWithTableMetadata()
     {
         createTable("CREATE TABLE %s(k int, c int, v1 int, v2 int, PRIMARY KEY(k,c))");
         createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c, v2) USING '%s' WITH OPTIONS = {'foo':'bar'}",
@@ -829,6 +840,7 @@ public class CustomIndexTest extends CQLTester
 
 
     // Used for index creation above
+    @SuppressWarnings("unused")
     public static class BrokenCustom2I extends StubIndex
     {
         public BrokenCustom2I(ColumnFamilyStore baseCfs, IndexMetadata metadata)
@@ -842,12 +854,10 @@ public class CustomIndexTest extends CQLTester
         }
     }
 
-    private void testCreateIndex(String indexName, String... targetColumnNames) throws Throwable
+    private void testCreateIndex(String indexName, String... targetColumnNames)
     {
         createIndex(String.format("CREATE CUSTOM INDEX %s ON %%s(%s) USING '%s'",
-                                  indexName,
-                                  Arrays.stream(targetColumnNames).collect(Collectors.joining(",")),
-                                  StubIndex.class.getName()));
+                                  indexName, String.join(",", targetColumnNames), StubIndex.class.getName()));
         assertIndexCreated(indexName, targetColumnNames);
     }
 
@@ -1023,6 +1033,7 @@ public class CustomIndexTest extends CQLTester
             super(baseCfs, metadata);
         }
 
+        @SuppressWarnings("unused")
         public static Map<String, String> validateOptions(Map<String, String> options)
         {
             IndexWithValidateOptions.options = options;
@@ -1040,6 +1051,7 @@ public class CustomIndexTest extends CQLTester
             super(baseCfs, metadata);
         }
 
+        @SuppressWarnings("unused")
         public static Map<String, String> validateOptions(Map<String, String> options, TableMetadata table)
         {
             IndexWithOverloadedValidateOptions.options = options;
@@ -1081,11 +1093,13 @@ public class CustomIndexTest extends CQLTester
         // When we're done indexing the partition, the test checks the states of the
         // various OpOrder.Groups, which it can obtain from this index.
 
+        @Override
         public Indexer indexerFor(final DecoratedKey key,
                                   RegularAndStaticColumns columns,
                                   int nowInSec,
                                   WriteContext ctx,
-                                  IndexTransaction.Type transactionType)
+                                  IndexTransaction.Type transactionType,
+                                  Memtable memtable)
         {
             CassandraWriteContext cassandraWriteContext = (CassandraWriteContext) ctx;
             if (readOrderingAtStart == null)
@@ -1134,4 +1148,529 @@ public class CustomIndexTest extends CQLTester
             };
         }
     }
+
+
+    @Test
+    public void testFlushObserver() throws Throwable
+    {
+        createTable("CREATE TABLE %s (k int, c int, s int static, v int, PRIMARY KEY (k, c))");
+        String indexName = "test_index_with_flush_observer";
+        createIndex(String.format("CREATE CUSTOM INDEX %s ON %%s(v) USING '%s'",
+                                  indexName, IndexWithFlushObserver.class.getName()));
+
+        execute("INSERT INTO %s (k, c, s, v) VALUES (?, ?, ?, ?)", 0, 0, 0, 0);
+        execute("INSERT INTO %s (k, c, s, v) VALUES (?, ?, ?, ?)", 0, 1, 1, 1);
+        execute("INSERT INTO %s (k, c, s, v) VALUES (?, ?, ?, ?)", 1, 0, 2, 2);
+        execute("INSERT INTO %s (k, c, s, v) VALUES (?, ?, ?, ?)", 1, 1, 3, 3);
+
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        SecondaryIndexManager indexManager = cfs.indexManager;
+        IndexWithFlushObserver index = (IndexWithFlushObserver) indexManager.getIndexByName(indexName);
+
+        assertEquals(0, index.beginFlushCalls.get());
+        assertEquals(0, index.flushedPartitions.get());
+        assertEquals(0, index.flushedStaticRows.get());
+        assertEquals(0, index.flushedUnfiltereds.get());
+        assertEquals(0, index.completeFlushCalls.get());
+
+        cfs.forceBlockingFlush(FlushReason.UNIT_TESTS);
+
+        assertEquals(1, index.beginFlushCalls.get());
+        assertEquals(2, index.flushedPartitions.get());
+        assertEquals(2, index.flushedStaticRows.get());
+        assertEquals(4, index.flushedUnfiltereds.get());
+        assertEquals(1, index.completeFlushCalls.get());
+
+        execute("DELETE FROM %s WHERE k=?", 0);
+        execute("DELETE FROM %s WHERE k=? AND c>=?", 1, 1);
+        index.reset();
+        cfs.forceBlockingFlush(FlushReason.UNIT_TESTS);
+
+        assertEquals(1, index.beginFlushCalls.get());
+        assertEquals(2, index.flushedPartitions.get());
+        assertEquals(0, index.flushedStaticRows.get()); // flushed data has no static values..
+        assertEquals(2, index.flushedUnfiltereds.get());
+        assertEquals(1, index.completeFlushCalls.get());
+    }
+
+    /**
+     * A {@link StubIndex} using a {@link SSTableFlushObserver} that just keeps count of operations.
+     */
+    public static final class IndexWithFlushObserver extends StubIndex
+    {
+
+        AtomicInteger beginFlushCalls = new AtomicInteger();
+        AtomicInteger flushedPartitions = new AtomicInteger();
+        AtomicInteger flushedStaticRows = new AtomicInteger();
+        AtomicInteger flushedUnfiltereds = new AtomicInteger();
+        AtomicInteger completeFlushCalls = new AtomicInteger();
+
+        public IndexWithFlushObserver(ColumnFamilyStore baseCfs, IndexMetadata metadata)
+        {
+            super(baseCfs, metadata);
+        }
+
+        @Override
+        public void reset()
+        {
+            super.reset();
+            beginFlushCalls.set(0);
+            flushedPartitions.set(0);
+            flushedStaticRows.set(0);
+            flushedUnfiltereds.set(0);
+            completeFlushCalls.set(0);
+        }
+
+        @Override
+        public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker)
+        {
+            return new SSTableFlushObserver() {
+
+                @Override
+                public void begin()
+                {
+                    beginFlushCalls.incrementAndGet();
+                }
+
+                @Override
+                public void startPartition(DecoratedKey key, long keyPosition, long keyPositionForSASI)
+                {
+                    flushedPartitions.incrementAndGet();
+                }
+
+                @Override
+                public void staticRow(Row staticRow)
+                {
+                    flushedStaticRows.incrementAndGet();
+                }
+
+                @Override
+                public void nextUnfilteredCluster(Unfiltered unfiltered)
+                {
+                    flushedUnfiltereds.incrementAndGet();
+                }
+
+                @Override
+                public void complete()
+                {
+                    completeFlushCalls.incrementAndGet();
+                }
+            };
+        }
+    }
+
+    /**
+     * Verify that writes for indexes in the same {@link Index.Group} are grouped.
+     */
+    @Test
+    public void testGroupedWrites() throws Throwable
+    {
+        // create the schema with two indexes in the same group
+        String indexClassName = IndexWithSharedGroup.class.getName();
+        createTable("CREATE TABLE %s (k int, c int, s int static, v int, PRIMARY KEY (k,c))");
+        createIndex(String.format("CREATE CUSTOM INDEX grouped_index_c ON %%s(c) USING '%s'", indexClassName));
+        createIndex(String.format("CREATE CUSTOM INDEX grouped_index_v ON %%s(v) USING '%s'", indexClassName));
+
+        // retrieve the indexes and their shared group
+        ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
+        SecondaryIndexManager indexManager = cfs.indexManager;
+        StubIndex index1 = (IndexWithSharedGroup) indexManager.getIndexByName("grouped_index_c");
+        StubIndex index2 = (IndexWithSharedGroup) indexManager.getIndexByName("grouped_index_v");
+        IndexWithSharedGroup.Group group = indexManager.listIndexGroups()
+                                                       .stream()
+                                                       .filter(g -> g instanceof IndexWithSharedGroup.Group)
+                                                       .map(g -> (IndexWithSharedGroup.Group) g)
+                                                       .findAny()
+                                                       .orElseThrow(AssertionError::new);
+
+        // verify that row insertions get to the index group and they are propagated to their members
+        execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", 0, 0, 0);
+        execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", 0, 1, 1);
+        execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", 1, 0, 3);
+        assertEquals(3, group.rowsInserted.get());
+        assertEquals(3, index1.rowsInserted.size());
+        assertEquals(3, index2.rowsInserted.size());
+
+        // verify that row updates get to the index group and they are propagated to their members
+        execute("UPDATE %s SET v=? WHERE k=? AND c=?", 10, 0, 0);
+        execute("UPDATE %s SET v=? WHERE k=? AND c=?", 10, 1, 0);
+        assertEquals(2, group.rowsUpdated.get());
+        assertEquals(2, index1.rowsUpdated.size());
+        assertEquals(2, index2.rowsUpdated.size());
+
+        // verify that partition deletions get to the index group and its members
+        ReadCommand cmd = Util.cmd(cfs, 0).build();
+        try (ReadExecutionController executionController = cmd.executionController();
+             UnfilteredPartitionIterator iterator = cmd.executeLocally(executionController))
+        {
+            assertTrue(iterator.hasNext());
+            cfs.indexManager.deletePartition(iterator.next(), FBUtilities.nowInSeconds());
+        }
+        assertEquals(1, group.partitionDeletions.get());
+        assertEquals(1, index1.partitionDeletions.size());
+        assertEquals(1, index2.partitionDeletions.size());
+
+        // verify that the row deletions produced by the previous partition deletion get to the group and its members
+        assertEquals(2, group.rowsDeleted.get());
+        assertEquals(2, index1.rowsDeleted.size());
+        assertEquals(2, index2.rowsDeleted.size());
+
+        // verify that range tombstones get to the index group and its members
+        execute("DELETE FROM %s WHERE k=? AND c>?", 0, 0);
+        execute("DELETE FROM %s WHERE k=? AND c>?", 1, 1);
+        assertEquals(2, group.rangeTombstones.get());
+        assertEquals(2, index1.rangeTombstones.size());
+        assertEquals(2, index2.rangeTombstones.size());
+
+        // verify the total number of begin calls
+        assertEquals(10, group.beginCalls.get());
+        assertEquals(10, index1.beginCalls);
+        assertEquals(10, index2.beginCalls);
+
+        // verify the total number of finish calls
+        assertEquals(10, group.finishCalls.get());
+        assertEquals(10, index1.finishCalls);
+        assertEquals(10, index2.finishCalls);
+
+        // flush the previous data to get rid of it, reset the group counters and flush a new memtable
+        cfs.forceBlockingFlush(FlushReason.UNIT_TESTS);
+        group.reset();
+        execute("INSERT INTO %s (k, s) VALUES (?, ?)", 1, 0);
+        execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", 1, 0, 0);
+        execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", 1, 1, 0);
+        execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", 2, 0, 0);
+        execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", 2, 1, 0);
+        execute("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)", 2, 2, 0);
+        execute("DELETE FROM %s WHERE k=? AND c=?", 2, 3);
+        execute("DELETE FROM %s WHERE k=?", 3);
+        cfs.forceBlockingFlush(FlushReason.UNIT_TESTS);
+
+        // verify that the flush observer calls get only once to the group
+        assertEquals(1, group.beginFlushCalls.get());
+        assertEquals(3, group.flushedPartitions.get());
+        assertEquals(3, group.flushedStaticRows.get());
+        assertEquals(6, group.flushedUnfiltereds.get());
+        assertEquals(1, group.completeFlushCalls.get());
+
+        // verify that the index rebuilds can be directed only to the first index
+        group.reset();
+        indexManager.rebuildIndexesBlocking(Collections.singleton(index1.getIndexMetadata().name));
+        assertEquals(8, group.rowsInserted.get());
+        assertEquals(8, index1.rowsInserted.size());
+        assertEquals(0, index2.rowsInserted.size());
+
+        // verify that the index rebuilds can be directed only to the second index
+        group.reset();
+        indexManager.rebuildIndexesBlocking(Collections.singleton(index2.getIndexMetadata().name));
+        assertEquals(8, group.rowsInserted.get());
+        assertEquals(0, index1.rowsInserted.size());
+        assertEquals(8, index2.rowsInserted.size());
+    }
+
+    @Test
+    public void testIndexGroupsInstancesManagement() throws Throwable
+    {
+        String indexClassName = IndexWithSharedGroup.class.getName();
+        createTable("CREATE TABLE %s (k int PRIMARY KEY, v1 int, v2 int, v3 int, v4 int, v5 int)");
+        SecondaryIndexManager indexManager = getCurrentColumnFamilyStore().indexManager;
+
+        // create two indexes belonging to the same group and verify that only one group is added to the manager
+        String idx1 = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v1) USING '%s'", indexClassName));
+        String idx2 = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v2) USING '%s'", indexClassName));
+        IndexWithSharedGroup.Group group = indexManager.listIndexGroups()
+                                                       .stream()
+                                                       .filter(g -> g instanceof IndexWithSharedGroup.Group)
+                                                       .map(g -> (IndexWithSharedGroup.Group) g)
+                                                       .findAny()
+                                                       .orElseThrow(AssertionError::new);
+
+        // verify that only one group has been added to the manager
+        assertEquals(2, indexManager.listIndexes().size());
+        assertEquals(1, indexManager.listIndexGroups().size());
+        assertEquals(2, group.indexes.size());
+
+        // create two indexes belonging to their own singleton group and verify that two groups are added to the manager
+        String idx3 = createIndex("CREATE INDEX ON %s(v3)");
+        String idx4 = createIndex("CREATE INDEX ON %s(v4)");
+        assertEquals(4, indexManager.listIndexes().size());
+        assertEquals(3, indexManager.listIndexGroups().size());
+
+        // create another index to the shared group and verify that they are added to the existing group instance
+        String idx5 = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(v5) USING '%s'", indexClassName));
+        assertEquals(5, indexManager.listIndexes().size());
+        assertEquals(3, indexManager.listIndexGroups().size());
+        assertEquals(3, group.indexes.size());
+
+        // drop one of the shared group members and verify that the manager still has the same group count
+        dropIndex("DROP INDEX %s." + idx1);
+        assertEquals(4, indexManager.listIndexes().size());
+        assertEquals(3, indexManager.listIndexGroups().size());
+        assertEquals(2, group.indexes.size());
+
+        // drop the standalone indexes and verify that their singleton groups are removed from the manager
+        dropIndex("DROP INDEX %s." + idx3);
+        dropIndex("DROP INDEX %s." + idx4);
+        assertEquals(2, indexManager.listIndexes().size());
+        assertEquals(1, indexManager.listIndexGroups().size());
+
+        // drop the remaining members of the shared group and verify that it is kept empty in the manager
+        dropIndex("DROP INDEX %s." + idx2);
+        dropIndex("DROP INDEX %s." + idx5);
+        assertEquals(0, indexManager.listIndexes().size());
+        assertEquals(1, indexManager.listIndexGroups().size());
+        assertEquals(0, group.indexes.size());
+
+        // create the sharing group members again and verify that they are added to the existing group instance
+        createIndex(String.format("CREATE CUSTOM INDEX %s ON %%s(v1) USING '%s'", idx1, indexClassName));
+        createIndex(String.format("CREATE CUSTOM INDEX %s ON %%s(v2) USING '%s'", idx2, indexClassName));
+        createIndex(String.format("CREATE CUSTOM INDEX %s ON %%s(v3) USING '%s'", idx3, indexClassName));
+        assertEquals(3, indexManager.listIndexes().size());
+        assertEquals(1, indexManager.listIndexGroups().size());
+        assertEquals(3, group.indexes.size());
+    }
+
+    /**
+     * {@link StubIndex} implementation that uses the same {@link Index.Group} for all its instances.
+     * That group keeps count of the calls and passes them to its members.
+     */
+    public static final class IndexWithSharedGroup extends StubIndex
+    {
+        public IndexWithSharedGroup(ColumnFamilyStore baseCfs, IndexMetadata metadata)
+        {
+            super(baseCfs, metadata);
+        }
+
+        @Override
+        public boolean shouldBuildBlocking()
+        {
+            return true;
+        }
+
+        @Override
+        public void register(IndexRegistry registry)
+        {
+            registry.registerIndex(this, Group.class, Group::new);
+        }
+
+        private static class Group implements Index.Group
+        {
+            Map<String, IndexWithSharedGroup> indexes = Maps.newConcurrentMap();
+
+            AtomicInteger beginCalls = new AtomicInteger();
+            AtomicInteger finishCalls = new AtomicInteger();
+            AtomicInteger partitionDeletions = new AtomicInteger();
+            AtomicInteger rangeTombstones = new AtomicInteger();
+            AtomicInteger rowsInserted = new AtomicInteger();
+            AtomicInteger rowsDeleted = new AtomicInteger();
+            AtomicInteger rowsUpdated = new AtomicInteger();
+
+            AtomicInteger beginFlushCalls = new AtomicInteger();
+            AtomicInteger flushedPartitions = new AtomicInteger();
+            AtomicInteger flushedStaticRows = new AtomicInteger();
+            AtomicInteger flushedUnfiltereds = new AtomicInteger();
+            AtomicInteger completeFlushCalls = new AtomicInteger();
+
+            public void reset()
+            {
+                beginCalls.set(0);
+                finishCalls.set(0);
+                partitionDeletions.set(0);
+                rangeTombstones.set(0);
+                rowsInserted.set(0);
+                rowsDeleted.set(0);
+                rowsUpdated.set(0);
+                beginFlushCalls.set(0);
+                flushedPartitions.set(0);
+                flushedStaticRows.set(0);
+                flushedUnfiltereds.set(0);
+                completeFlushCalls.set(0);
+                indexes.values().forEach(IndexWithSharedGroup::reset);
+            }
+
+            @Override
+            public Set<Index> getIndexes()
+            {
+                return ImmutableSet.copyOf(indexes.values());
+            }
+
+            @Override
+            public void addIndex(Index index)
+            {
+                indexes.put(index.getIndexMetadata().name, (IndexWithSharedGroup) index);
+            }
+
+            @Override
+            public void removeIndex(Index index)
+            {
+                indexes.remove(index.getIndexMetadata().name);
+            }
+
+            @Override
+            public boolean containsIndex(Index index)
+            {
+                return indexes.containsKey(index.getIndexMetadata().name);
+            }
+
+            @Override
+            public Index.Indexer indexerFor(Predicate<Index> indexSelector,
+                                            DecoratedKey key,
+                                            RegularAndStaticColumns columns,
+                                            int nowInSec,
+                                            WriteContext context,
+                                            IndexTransaction.Type transactionType,
+                                            Memtable memtable)
+            {
+                Set<Index.Indexer> indexers = indexes.values()
+                                                     .stream()
+                                                     .filter(indexSelector)
+                                                     .map(i -> i.indexerFor(key, columns, nowInSec, context, transactionType, memtable))
+                                                     .filter(Objects::nonNull)
+                                                     .collect(Collectors.toSet());
+
+                return indexers.isEmpty() ? null : new Index.Indexer() {
+
+                    @Override
+                    public void begin()
+                    {
+                        beginCalls.incrementAndGet();
+                        indexers.forEach(Indexer::begin);
+                    }
+
+                    @Override
+                    public void partitionDelete(DeletionTime deletionTime)
+                    {
+                        partitionDeletions.incrementAndGet();
+                        indexers.forEach(indexer -> indexer.partitionDelete(deletionTime));
+                    }
+
+                    @Override
+                    public void rangeTombstone(RangeTombstone tombstone)
+                    {
+                        rangeTombstones.incrementAndGet();
+                        indexers.forEach(indexer -> indexer.rangeTombstone(tombstone));
+                    }
+
+                    @Override
+                    public void insertRow(Row row)
+                    {
+                        rowsInserted.incrementAndGet();
+                        indexers.forEach(indexer -> indexer.insertRow(row));
+                    }
+
+                    @Override
+                    public void removeRow(Row row)
+                    {
+                        rowsDeleted.incrementAndGet();
+                        indexers.forEach(indexer -> indexer.removeRow(row));
+                    }
+
+                    @Override
+                    public void updateRow(Row oldRow, Row newRow)
+                    {
+                        rowsUpdated.incrementAndGet();
+                        indexers.forEach(indexer -> indexer.updateRow(oldRow, newRow));
+                    }
+
+                    @Override
+                    public void finish()
+                    {
+                        finishCalls.incrementAndGet();
+                        indexers.forEach(Indexer::finish);
+                    }
+                };
+            }
+
+            @Override
+            public QueryPlan queryPlanFor(RowFilter rowFilter)
+            {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata)
+            {
+                Set<SSTableFlushObserver> observers = indexes.values()
+                                                             .stream()
+                                                             .map(i -> i.getFlushObserver(descriptor, tracker))
+                                                             .filter(Objects::nonNull)
+                                                             .collect(Collectors.toSet());
+
+                return new SSTableFlushObserver() {
+
+                    @Override
+                    public void begin()
+                    {
+                        beginFlushCalls.incrementAndGet();
+                        observers.forEach(SSTableFlushObserver::begin);
+                    }
+
+                    @Override
+                    public void startPartition(DecoratedKey key, long position, long keyPositionForSASI)
+                    {
+                        flushedPartitions.incrementAndGet();
+                        observers.forEach(o -> o.startPartition(key, position, keyPositionForSASI));
+                    }
+
+                    @Override
+                    public void staticRow(Row staticRow)
+                    {
+                        flushedStaticRows.incrementAndGet();
+                        observers.forEach(o -> o.staticRow(staticRow));
+                    }
+
+                    @Override
+                    public void nextUnfilteredCluster(Unfiltered unfiltered)
+                    {
+                        flushedUnfiltereds.incrementAndGet();
+                        observers.forEach(o -> o.nextUnfilteredCluster(unfiltered));
+                    }
+
+                    @Override
+                    public void complete()
+                    {
+                        completeFlushCalls.incrementAndGet();
+                        observers.forEach(SSTableFlushObserver::complete);
+                    }
+                };
+            }
+
+            @Override
+            public Set<Component> getComponents()
+            {
+                return Collections.emptySet();
+            }
+        }
+    }
+
+    @Test
+    public void testMulticolumnIndexWithBaseTable() throws Throwable
+    {
+        createTable("CREATE TABLE %s(k int PRIMARY KEY, v int)");
+        assertInvalidMessage("Indexes belonging to a group of indexes shouldn't have a backing table",
+                             String.format("CREATE CUSTOM INDEX ON %%s(v) USING '%s'",
+                                           MulticolumnIndexWithBaseTable.class.getName()));
+    }
+
+    public static final class MulticolumnIndexWithBaseTable extends StubIndex
+    {
+        private final ColumnFamilyStore baseCfs;
+
+        public MulticolumnIndexWithBaseTable(ColumnFamilyStore baseCfs, IndexMetadata metadata)
+        {
+            super(baseCfs, metadata);
+            this.baseCfs = baseCfs;
+        }
+
+        @Override
+        public void register(IndexRegistry registry)
+        {
+            registry.registerIndex(this, MulticolumnIndexWithBaseTable.class, StubIndexGroup::new);
+        }
+
+        @Override
+        public Optional<ColumnFamilyStore> getBackingTable()
+        {
+            return Optional.of(baseCfs);
+        }
+    }    
 }
diff --git a/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java b/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java
index d8fb99f40f..c9dde2ae69 100644
--- a/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java
+++ b/test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java
@@ -766,7 +766,7 @@ public class SecondaryIndexManagerTest extends CQLTester
         {
             return new CollatedViewIndexBuildingSupport()
             {
-                public SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs, Set<Index> indexes, Collection<SSTableReader> sstables)
+                public SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs, Set<Index> indexes, Collection<SSTableReader> sstables, boolean isFullRebuild)
                 {
                     try
                     {
@@ -775,7 +775,7 @@ public class SecondaryIndexManagerTest extends CQLTester
                             buildWaitLatch.countDown();
                             buildLatch.await();
                         }
-                        final SecondaryIndexBuilder builder = super.getIndexBuildTask(cfs, indexes, sstables);
+                        final SecondaryIndexBuilder builder = super.getIndexBuildTask(cfs, indexes, sstables, isFullRebuild);
                         return new SecondaryIndexBuilder()
                         {
 
diff --git a/test/unit/org/apache/cassandra/index/StubIndex.java b/test/unit/org/apache/cassandra/index/StubIndex.java
index 02ccbff134..623b08def9 100644
--- a/test/unit/org/apache/cassandra/index/StubIndex.java
+++ b/test/unit/org/apache/cassandra/index/StubIndex.java
@@ -20,16 +20,16 @@ package org.apache.cassandra.index;
 
 import java.util.*;
 import java.util.concurrent.Callable;
-import java.util.function.BiFunction;
 
 import org.apache.cassandra.Util;
+import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
 import org.apache.cassandra.schema.ColumnMetadata;
 import org.apache.cassandra.cql3.Operator;
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.filter.RowFilter;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -100,7 +100,8 @@ public class StubIndex implements Index
                               RegularAndStaticColumns columns,
                               int nowInSec,
                               WriteContext ctx,
-                              IndexTransaction.Type transactionType)
+                              IndexTransaction.Type transactionType,
+                              Memtable memtable)
     {
         return new Indexer()
         {
@@ -205,11 +206,28 @@ public class StubIndex implements Index
 
     public Searcher searcherFor(final ReadCommand command)
     {
-        return (controller) -> Util.executeLocally((PartitionRangeReadCommand)command, baseCfs, controller);
+        return new Searcher(command);
     }
 
-    public BiFunction<PartitionIterator, ReadCommand, PartitionIterator> postProcessorFor(ReadCommand readCommand)
+    protected class Searcher implements Index.Searcher
     {
-        return (iter, command) -> iter;
+        private final ReadCommand command;
+
+        Searcher(ReadCommand command)
+        {
+            this.command = command;
+        }
+
+        @Override
+        public ReadCommand command()
+        {
+            return command;
+        }
+
+        @Override
+        public UnfilteredPartitionIterator search(ReadExecutionController executionController)
+        {
+            return Util.executeLocally((PartitionRangeReadCommand)command, baseCfs, executionController);
+        }
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/StubIndexGroup.java b/test/unit/org/apache/cassandra/index/StubIndexGroup.java
new file mode 100644
index 0000000000..5e4bc299af
--- /dev/null
+++ b/test/unit/org/apache/cassandra/index/StubIndexGroup.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.index;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.function.Predicate;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.RegularAndStaticColumns;
+import org.apache.cassandra.db.WriteContext;
+import org.apache.cassandra.db.filter.RowFilter;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.memtable.Memtable;
+import org.apache.cassandra.index.transactions.IndexTransaction;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableFlushObserver;
+import org.apache.cassandra.schema.TableMetadata;
+
+/**
+ * Basic custom index group implementation for testing.
+ */
+public class StubIndexGroup implements Index.Group
+{
+    private final Set<Index> indexes = new HashSet<>();
+
+    @Override
+    public Set<Index> getIndexes()
+    {
+        return indexes;
+    }
+
+    @Override
+    public void addIndex(Index index)
+    {
+        indexes.add(index);
+    }
+
+    @Override
+    public void removeIndex(Index index)
+    {
+        indexes.remove(index);
+    }
+
+    @Override
+    public boolean containsIndex(Index index)
+    {
+        return indexes.contains(index);
+    }
+
+    @Override
+    public Index.Indexer indexerFor(Predicate<Index> indexSelector,
+                                    DecoratedKey key,
+                                    RegularAndStaticColumns columns,
+                                    int nowInSec,
+                                    WriteContext context,
+                                    IndexTransaction.Type transactionType,
+                                    Memtable memtable)
+    {
+        return null;
+    }
+
+    @Nullable
+    @Override
+    public Index.QueryPlan queryPlanFor(RowFilter rowFilter)
+    {
+        return null;
+    }
+
+    @Override
+    public SSTableFlushObserver getFlushObserver(Descriptor descriptor, LifecycleNewTracker tracker, TableMetadata tableMetadata)
+    {
+        return null;
+    }
+
+    public Set<Component> getComponents()
+    {
+        return Collections.emptySet();
+    }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java b/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java
index 04579b0399..9e1a5d82ee 100644
--- a/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java
+++ b/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java
@@ -24,7 +24,6 @@ import java.nio.ByteBuffer;
 import java.util.*;
 import java.util.concurrent.Callable;
 import java.util.concurrent.Future;
-import java.util.function.BiFunction;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 
@@ -46,7 +45,7 @@ import org.apache.cassandra.db.filter.RowFilter;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
 import org.apache.cassandra.db.lifecycle.View;
 import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.partitions.PartitionIterator;
+import org.apache.cassandra.db.memtable.Memtable;
 import org.apache.cassandra.db.partitions.PartitionUpdate;
 import org.apache.cassandra.db.rows.*;
 import org.apache.cassandra.exceptions.InvalidRequestException;
@@ -54,7 +53,6 @@ import org.apache.cassandra.index.Index;
 import org.apache.cassandra.index.IndexRegistry;
 import org.apache.cassandra.index.SecondaryIndexBuilder;
 import org.apache.cassandra.index.transactions.IndexTransaction;
-import org.apache.cassandra.index.transactions.UpdateTransaction;
 import org.apache.cassandra.io.sstable.ReducingKeyIterator;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.IndexMetadata;
@@ -210,14 +208,6 @@ public class CustomCassandraIndex implements Index
         return indexCfs.getMeanEstimatedCellPerPartitionCount();
     }
 
-    /**
-     * No post processing of query results, just return them unchanged
-     */
-    public BiFunction<PartitionIterator, ReadCommand, PartitionIterator> postProcessorFor(ReadCommand command)
-    {
-        return (partitionIterator, readCommand) -> partitionIterator;
-    }
-
     public RowFilter getPostIndexQueryFilter(RowFilter filter)
     {
         return getTargetExpression(filter.getExpressions()).map(filter::without)
@@ -290,7 +280,8 @@ public class CustomCassandraIndex implements Index
                               final RegularAndStaticColumns columns,
                               final int nowInSec,
                               final WriteContext ctx,
-                              final IndexTransaction.Type transactionType)
+                              final IndexTransaction.Type transactionType,
+                              final Memtable memtable)
     {
         if (!isPrimaryKeyIndex() && !columns.contains(indexedColumn))
             return null;
@@ -462,7 +453,7 @@ public class CustomCassandraIndex implements Index
                                                                cell));
         Row row = BTreeRow.noCellLiveRow(buildIndexClustering(rowKey, clustering, cell), info);
         PartitionUpdate upd = partitionUpdate(valueKey, row);
-        indexCfs.getWriteHandler().write(upd, ctx, UpdateTransaction.NO_OP);
+        indexCfs.getWriteHandler().write(upd, ctx, false);
         logger.debug("Inserted entry into index for value {}", valueKey);
     }
 
@@ -508,7 +499,7 @@ public class CustomCassandraIndex implements Index
     {
         Row row = BTreeRow.emptyDeletedRow(indexClustering, Row.Deletion.regular(deletion));
         PartitionUpdate upd = partitionUpdate(indexKey, row);
-        indexCfs.getWriteHandler().write(upd, ctx, UpdateTransaction.NO_OP);
+        indexCfs.getWriteHandler().write(upd, ctx, false);
         logger.debug("Removed index entry for value {}", indexKey);
     }
 
diff --git a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
index 3c33327c97..fd13659535 100644
--- a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java
@@ -110,7 +110,7 @@ import org.apache.cassandra.index.sasi.disk.Token;
 import org.apache.cassandra.index.sasi.exceptions.TimeQuotaExceededException;
 import org.apache.cassandra.index.sasi.memory.IndexMemtable;
 import org.apache.cassandra.index.sasi.plan.QueryController;
-import org.apache.cassandra.index.sasi.plan.QueryPlan;
+import org.apache.cassandra.index.sasi.plan.SASIIndexSearcher;
 import org.apache.cassandra.index.sasi.utils.RangeIterator;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
@@ -1506,7 +1506,7 @@ public class SASIIndexTest
                                              DataRange.allData(store.metadata().partitioner));
         try
         {
-            new QueryPlan(store, command, 0).execute(ReadExecutionController.empty());
+            new SASIIndexSearcher(store, command, 0).search(ReadExecutionController.empty());
             Assert.fail();
         }
         catch (TimeQuotaExceededException e)
@@ -1523,7 +1523,7 @@ public class SASIIndexTest
 
         try (ReadExecutionController controller = command.executionController())
         {
-            Set<String> rows = getKeys(new QueryPlan(store, command, DatabaseDescriptor.getRangeRpcTimeout(MILLISECONDS)).execute(controller));
+            Set<String> rows = getKeys(new SASIIndexSearcher(store, command, DatabaseDescriptor.getRangeRpcTimeout(MILLISECONDS)).search(controller));
             assertRows(rows, "key1", "key2", "key3", "key4");
         }
     }
diff --git a/test/unit/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriterTest.java b/test/unit/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriterTest.java
index 6707aa592e..02e87b374c 100644
--- a/test/unit/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriterTest.java
+++ b/test/unit/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriterTest.java
@@ -41,6 +41,7 @@ import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.marshal.Int32Type;
 import org.apache.cassandra.db.marshal.LongType;
 import org.apache.cassandra.db.marshal.UTF8Type;
@@ -93,7 +94,7 @@ public class PerSSTableIndexWriterTest extends SchemaLoader
 
         File directory = cfs.getDirectories().getDirectoryForNewSSTables();
         Descriptor descriptor = cfs.newSSTableDescriptor(directory);
-        PerSSTableIndexWriter indexWriter = (PerSSTableIndexWriter) sasi.getFlushObserver(descriptor, OperationType.FLUSH);
+        PerSSTableIndexWriter indexWriter = (PerSSTableIndexWriter) sasi.getFlushObserver(descriptor, LifecycleTransaction.offline(OperationType.FLUSH));
 
         SortedMap<DecoratedKey, Row> expectedKeys = new TreeMap<>(DecoratedKey.comparator);
 
@@ -187,7 +188,7 @@ public class PerSSTableIndexWriterTest extends SchemaLoader
 
         File directory = cfs.getDirectories().getDirectoryForNewSSTables();
         Descriptor descriptor = cfs.newSSTableDescriptor(directory);
-        PerSSTableIndexWriter indexWriter = (PerSSTableIndexWriter) sasi.getFlushObserver(descriptor, OperationType.FLUSH);
+        PerSSTableIndexWriter indexWriter = (PerSSTableIndexWriter) sasi.getFlushObserver(descriptor, LifecycleTransaction.offline(OperationType.FLUSH));
 
         final long now = System.currentTimeMillis();
 
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java
index 58ca05ba0f..fbeabb95cb 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableFlushObserverTest.java
@@ -107,10 +107,10 @@ public class SSTableFlushObserverTest
                                          .setMetadataCollector(new MetadataCollector(cfm.comparator).sstableLevel(0))
                                          .setSerializationHeader(new SerializationHeader(true, cfm, cfm.regularAndStaticColumns(), EncodingStats.NO_STATS))
                                          .setFlushObservers(Collections.singletonList(observer))
-                                         .addDefaultComponents()
+                                         .addDefaultComponents(Collections.emptySet())
                                          .build(transaction, null);
 
-        SSTableReader reader = null;
+        SSTableReader reader;
         Multimap<ByteBuffer, Cell<?>> expected = ArrayListMultimap.create();
 
         try
@@ -151,7 +151,6 @@ public class SSTableFlushObserverTest
         for (Triple<ByteBuffer, Long, Long> e : observer.rows.keySet())
         {
             ByteBuffer key = e.getLeft();
-            long dataPosition = e.getMiddle();
             long indexPosition = e.getRight();
 
             DecoratedKey indexKey = reader.keyAtPositionFromSecondaryIndex(indexPosition);
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
index 5a82bc37db..921ffb112d 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableReaderTest.java
@@ -1060,7 +1060,7 @@ public class SSTableReaderTest
                                              .columns("birthdate")
                                              .filterOn("birthdate", Operator.EQ, 1L)
                                              .build();
-        Index.Searcher searcher = rc.getIndex(indexedCFS).searcherFor(rc);
+        Index.Searcher searcher = rc.indexSearcher();
         assertNotNull(searcher);
         try (ReadExecutionController executionController = rc.executionController())
         {
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
index 91b44ed27e..0cdd8ac18b 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableWriterTestBase.java
@@ -163,9 +163,9 @@ public class SSTableWriterTestBase extends SchemaLoader
                    .setPendingRepair(pendingRepair)
                    .setTransientSSTable(isTransient)
                    .setSerializationHeader(new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS))
-                   .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexes(), txn.opType())
+                   .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexGroups(), txn, cfs.metadata.get())
                    .setMetadataCollector(new MetadataCollector(cfs.metadata().comparator))
-                   .addDefaultComponents()
+                   .addDefaultComponents(cfs.indexManager.listIndexGroups())
                    .build(txn, cfs);
     }
 
diff --git a/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java b/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java
index c756d9a81b..bd5b4a5c18 100644
--- a/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/ScrubTest.java
@@ -840,7 +840,7 @@ public class ScrubTest
                                          .setMetadataCollector(collector)
                                          .setSerializationHeader(header)
                                          .setFlushObservers(Collections.emptyList())
-                                         .addDefaultComponents()
+                                         .addDefaultComponents(Collections.emptySet())
                                          .build(txn, cfs);
 
         return new TestMultiWriter(writer, txn);
diff --git a/test/unit/org/apache/cassandra/metrics/ClientRequestMetricsTest.java b/test/unit/org/apache/cassandra/metrics/ClientRequestMetricsTest.java
index 650bd95637..b4e9b085d2 100644
--- a/test/unit/org/apache/cassandra/metrics/ClientRequestMetricsTest.java
+++ b/test/unit/org/apache/cassandra/metrics/ClientRequestMetricsTest.java
@@ -24,6 +24,7 @@ import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import com.codahale.metrics.Histogram;
 import com.datastax.driver.core.BatchStatement;
 import com.datastax.driver.core.BoundStatement;
 import com.datastax.driver.core.Cluster;
@@ -33,10 +34,13 @@ import org.apache.cassandra.ServerTestUtils;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.service.EmbeddedCassandraService;
+import org.apache.cassandra.service.reads.range.RangeCommandIterator;
 
-import static com.datastax.driver.core.Cluster.builder;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
 
+import static com.datastax.driver.core.Cluster.builder;
+
 public class ClientRequestMetricsTest
 {
     private static Cluster cluster;
@@ -161,6 +165,23 @@ public class ClientRequestMetricsTest
         assertEquals(0, writeMetricsContainer.compareRemoteRequest());
     }
 
+    @Test
+    public void testRangeRead() throws Throwable
+    {
+        clearHistogram(RangeCommandIterator.rangeMetrics.roundTrips);
+        long latencyCount = RangeCommandIterator.rangeMetrics.latency.getCount();
+
+        session.execute("SELECT * FROM system.peers");
+
+        assertThat(RangeCommandIterator.rangeMetrics.roundTrips.getCount()).isGreaterThan(0);
+        assertThat(RangeCommandIterator.rangeMetrics.roundTrips.getSnapshot().getMax()).isEqualTo(1);
+        assertThat(RangeCommandIterator.rangeMetrics.latency.getCount()).isEqualTo(latencyCount + 1);
+    }
+
+    private void clearHistogram(Histogram histogram)
+    {
+        ((ClearableHistogram) histogram).clear();
+    }
 
     private static class ClientRequestMetricsContainer
     {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


[cassandra] 06/06: post-rebase fixes for the rebase on trunk at fad1f7457032544ab6a7b40c5d38ecb8b25899bb

Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

maedhroz pushed a commit to branch cep-7-sai
in repository https://gitbox.apache.org/repos/asf/cassandra.git

commit 6ac577308c37ed6b3c8c4400048725223ce1112a
Author: Caleb Rackliffe <ca...@gmail.com>
AuthorDate: Thu Jun 1 14:43:20 2023 -0500

    post-rebase fixes for the rebase on trunk at fad1f7457032544ab6a7b40c5d38ecb8b25899bb
---
 .../config/CassandraRelevantProperties.java        | 16 +++++---
 .../cassandra/config/DatabaseDescriptor.java       |  1 +
 .../apache/cassandra/index/IndexStatusManager.java | 18 ++++-----
 .../apache/cassandra/index/sai/QueryContext.java   |  4 +-
 .../index/sai/disk/v1/RowAwarePrimaryKeyMap.java   |  6 +--
 .../index/sai/disk/v1/SSTableIndexWriter.java      |  7 +++-
 .../sai/disk/v1/sortedterms/SortedTermsWriter.java |  2 +-
 .../index/sai/disk/v1/trie/TriePrefixSearcher.java |  4 +-
 .../apache/cassandra/index/sai/utils/TypeUtil.java |  6 +--
 .../io/sstable/format/bti/BtiTableWriter.java      |  6 ++-
 .../org/apache/cassandra/io/tries/TrieNode.java    |  2 +-
 .../apache/cassandra/io/tries/ValueIterator.java   |  8 ++--
 .../org/apache/cassandra/index/sai/SAITester.java  | 13 ++++--
 .../sai/disk/v1/InvertedIndexSearcherTest.java     |  2 +-
 .../index/sai/disk/v1/SegmentFlushTest.java        |  2 +-
 .../index/sai/disk/v1/TermsReaderTest.java         |  4 +-
 .../cassandra/index/sai/disk/v1/TermsScanner.java  |  4 +-
 .../sai/disk/v1/sortedterms/SortedTermsTest.java   | 16 ++++----
 .../sai/disk/v1/trie/TriePrefixSearcherTest.java   | 46 +++++++++++-----------
 .../sai/disk/v1/trie/TrieTermsDictionaryTest.java  |  8 ++--
 .../index/sai/memory/RAMStringIndexerTest.java     |  2 +-
 .../index/sai/memory/TrieMemoryIndexTest.java      |  2 +-
 .../cassandra/index/sai/plan/OperationTest.java    |  3 +-
 .../index/sai/utils/AbstractPrimaryKeyTester.java  |  2 +-
 .../cassandra/index/sai/utils/TypeUtilTest.java    |  2 +-
 .../index/sai/virtual/SegmentsSystemViewTest.java  |  4 +-
 .../org/apache/cassandra/io/tries/WalkerTest.java  |  1 -
 .../apache/cassandra/io/util/SizedIntsTest.java    |  1 -
 .../io/util/TailOverridingRebuffererTest.java      |  4 +-
 29 files changed, 105 insertions(+), 91 deletions(-)

diff --git a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
index 99dc65cdb6..a81bcd9ed1 100644
--- a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
+++ b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java
@@ -390,16 +390,20 @@ public enum CassandraRelevantProperties
     RESET_BOOTSTRAP_PROGRESS("cassandra.reset_bootstrap_progress"),
     RING_DELAY("cassandra.ring_delay_ms"),
     /** Defines how often schema definitions are pulled from the other nodes */
-    SCHEMA_PULL_INTERVAL_MS("cassandra.schema_pull_interval_ms", "60000"),
-    SCHEMA_UPDATE_HANDLER_FACTORY_CLASS("cassandra.schema.update_handler_factory.class"),
-    SEARCH_CONCURRENCY_FACTOR("cassandra.search_concurrency_factor", "1"),
 
     // SAI specific properties
-    /** Latest version to be used for SAI index writing */
-    SAI_LATEST_VERSION("cassandra.sai.latest_version", "aa"),
 
     /** Controls the maximum number of index query intersections that will take part in a query */
     SAI_INTERSECTION_CLAUSE_LIMIT("cassandra.sai.intersection.clause.limit", "2"),
+    /** Latest version to be used for SAI index writing */
+    SAI_LATEST_VERSION("cassandra.sai.latest_version", "aa"),
+    SAI_MAX_FROZEN_TERM_SIZE("cassandra.sai.max_frozen_term_size_kb", "5"),
+    SAI_MAX_STRING_TERM_SIZE("cassandra.sai.max_string_term_size_kb", "1"),
+    SAI_TEST_DISABLE_TIMEOUT("cassandra.sai.test.disable.timeout", "false"),
+
+    SCHEMA_PULL_INTERVAL_MS("cassandra.schema_pull_interval_ms", "60000"),
+    SCHEMA_UPDATE_HANDLER_FACTORY_CLASS("cassandra.schema.update_handler_factory.class"),
+    SEARCH_CONCURRENCY_FACTOR("cassandra.search_concurrency_factor", "1"),
 
     /**
      * The maximum number of seeds returned by a seed provider before emmitting a warning.
@@ -461,6 +465,7 @@ public enum CassandraRelevantProperties
     TEST_DEBUG_REF_COUNT("cassandra.debugrefcount"),
     TEST_DRIVER_CONNECTION_TIMEOUT_MS("cassandra.test.driver.connection_timeout_ms", "5000"),
     TEST_DRIVER_READ_TIMEOUT_MS("cassandra.test.driver.read_timeout_ms", "12000"),
+    TEST_ENCRYPTION("cassandra.test.encryption", "false"),
     TEST_FAIL_MV_LOCKS_COUNT("cassandra.test.fail_mv_locks_count", "0"),
     TEST_FAIL_WRITES_KS("cassandra.test.fail_writes_ks", ""),
     /** Flush changes of {@link org.apache.cassandra.schema.SchemaKeyspace} after each schema modification. In production,
@@ -473,6 +478,7 @@ public enum CassandraRelevantProperties
     TEST_JVM_DTEST_DISABLE_SSL("cassandra.test.disable_ssl"),
     TEST_LEGACY_SSTABLE_ROOT("legacy-sstable-root"),
     TEST_ORG_CAFFINITAS_OHC_SEGMENTCOUNT("org.caffinitas.ohc.segmentCount"),
+    TEST_RANDOM_SEED("cassandra.test.random.seed"),
     TEST_READ_ITERATION_DELAY_MS("cassandra.test.read_iteration_delay_ms", "0"),
     TEST_REUSE_PREPARED("cassandra.test.reuse_prepared", "true"),
     TEST_ROW_CACHE_SIZE("cassandra.test.row_cache_size"),
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 84d85e8e46..0e8756cec0 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -37,6 +37,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.OptionalDouble;
 import java.util.ServiceLoader;
 import java.util.Set;
 import java.util.UUID;
diff --git a/src/java/org/apache/cassandra/index/IndexStatusManager.java b/src/java/org/apache/cassandra/index/IndexStatusManager.java
index 0a872b5a0d..13a51831f9 100644
--- a/src/java/org/apache/cassandra/index/IndexStatusManager.java
+++ b/src/java/org/apache/cassandra/index/IndexStatusManager.java
@@ -31,7 +31,6 @@ import com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.cassandra.concurrent.ExecutorPlus;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.Keyspace;
@@ -42,19 +41,19 @@ import org.apache.cassandra.gms.Gossiper;
 import org.apache.cassandra.gms.VersionedValue;
 import org.apache.cassandra.locator.Endpoints;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.serializers.MarshalException;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.utils.FBUtilities;
-import org.json.simple.JSONValue;
-import org.json.simple.parser.ParseException;
+import org.apache.cassandra.utils.JsonUtils;
 
 import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
 
 /**
  * Handles the status of an index across the ring, updating the status per index and endpoint
  * in a per-endpoint map.
- *
+ * <p>
  * Peer status changes are recieved via the {@link StorageService} {@link org.apache.cassandra.gms.IEndpointStateChangeSubscriber}.
- *
+ * <p>
  * Local status changes are propagated to the {@link Gossiper} using an async executor.
  */
 public class IndexStatusManager
@@ -66,8 +65,6 @@ public class IndexStatusManager
     // executes index status propagation task asynchronously to avoid potential deadlock on SIM
     private final ExecutorPlus statusPropagationExecutor = executorFactory().withJmxInternal()
                                                                             .sequential("StatusPropagationExecutor");
-    // used to produce a status string from the current endpoint states in JSON format for Gossip propagation
-    private final ObjectMapper objectMapper = new ObjectMapper();
 
     /**
      * A map of per-endpoint index statuses: the key of inner map is the identifier "keyspace.index"
@@ -135,8 +132,7 @@ public class IndexStatusManager
             if (endpoint.equals(FBUtilities.getBroadcastAddressAndPort()))
                 return;
 
-            @SuppressWarnings("unchecked")
-            Map<String, String> peerStatus = (Map<String, String>) JSONValue.parseWithException(versionedValue.value);
+            Map<String, String> peerStatus = JsonUtils.fromJsonMap(versionedValue.value);
             Map<String, Index.Status> indexStatus = new HashMap<>();
 
             for (Map.Entry<String, String> e : peerStatus.entrySet())
@@ -153,7 +149,7 @@ public class IndexStatusManager
                 logger.debug("Received index status for peer {}:\n    Updated: {}\n    Removed: {}",
                              endpoint, updated, removed);
         }
-        catch (ParseException | IllegalArgumentException e)
+        catch (MarshalException | IllegalArgumentException e)
         {
             logger.warn("Unable to parse index status: {}", e.getMessage());
         }
@@ -185,7 +181,7 @@ public class IndexStatusManager
             // logged and this causes a number of dtests to fail.
             if (Gossiper.instance.isEnabled())
             {
-                String newStatus = objectMapper.writeValueAsString(states);
+                String newStatus = JsonUtils.JSON_OBJECT_MAPPER.writeValueAsString(states);
                 statusPropagationExecutor.submit(() -> {
                     // schedule gossiper update asynchronously to avoid potential deadlock when another thread is holding
                     // gossiper taskLock.
diff --git a/src/java/org/apache/cassandra/index/sai/QueryContext.java b/src/java/org/apache/cassandra/index/sai/QueryContext.java
index 4996acbd97..6bb85df219 100644
--- a/src/java/org/apache/cassandra/index/sai/QueryContext.java
+++ b/src/java/org/apache/cassandra/index/sai/QueryContext.java
@@ -25,6 +25,8 @@ import org.apache.cassandra.db.ReadCommand;
 import org.apache.cassandra.exceptions.QueryCancelledException;
 import org.apache.cassandra.utils.Clock;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_TEST_DISABLE_TIMEOUT;
+
 /**
  * Tracks state relevant to the execution of a single query, including metrics and timeout monitoring.
  *
@@ -33,7 +35,7 @@ import org.apache.cassandra.utils.Clock;
 @NotThreadSafe
 public class QueryContext
 {
-    private static final boolean DISABLE_TIMEOUT = Boolean.getBoolean("cassandra.sai.test.disable.timeout");
+    private static final boolean DISABLE_TIMEOUT = SAI_TEST_DISABLE_TIMEOUT.getBoolean();
 
     private final ReadCommand readCommand;
     private final long queryStartTimeNanos;
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/RowAwarePrimaryKeyMap.java b/src/java/org/apache/cassandra/index/sai/disk/v1/RowAwarePrimaryKeyMap.java
index d06969f21a..1319006d21 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/RowAwarePrimaryKeyMap.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/RowAwarePrimaryKeyMap.java
@@ -158,7 +158,7 @@ public class RowAwarePrimaryKeyMap implements PrimaryKeyMap
     @Override
     public long rowIdFromPrimaryKey(PrimaryKey key)
     {
-        return triePrefixSearcher.prefixSearch(key.asComparableBytes(ByteComparable.Version.OSS42));
+        return triePrefixSearcher.prefixSearch(key.asComparableBytes(ByteComparable.Version.OSS50));
     }
 
     @Override
@@ -172,10 +172,10 @@ public class RowAwarePrimaryKeyMap implements PrimaryKeyMap
         try
         {
             cursor.seekToPointId(sstableRowId);
-            ByteSource.Peekable peekable = ByteSource.peekable(cursor.term().asComparableBytes(ByteComparable.Version.OSS42));
+            ByteSource.Peekable peekable = ByteSource.peekable(cursor.term().asComparableBytes(ByteComparable.Version.OSS50));
 
             Token token = partitioner.getTokenFactory().fromComparableBytes(ByteSourceInverse.nextComponentSource(peekable),
-                                                                            ByteComparable.Version.OSS42);
+                                                                            ByteComparable.Version.OSS50);
             byte[] keyBytes = ByteSourceInverse.getUnescapedBytes(ByteSourceInverse.nextComponentSource(peekable));
 
             if (keyBytes == null)
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java
index fc12c41c85..8248bd80ac 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/SSTableIndexWriter.java
@@ -46,6 +46,9 @@ import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.NoSpamLogger;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_MAX_FROZEN_TERM_SIZE;
+import static org.apache.cassandra.config.CassandraRelevantProperties.SAI_MAX_STRING_TERM_SIZE;
+
 /**
  * Column index writer that accumulates (on-heap) indexed data from a compacted SSTable as it's being flushed to disk.
  */
@@ -55,8 +58,8 @@ public class SSTableIndexWriter implements PerColumnIndexWriter
     private static final Logger logger = LoggerFactory.getLogger(SSTableIndexWriter.class);
     private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1, TimeUnit.MINUTES);
 
-    public static final int MAX_STRING_TERM_SIZE = Integer.getInteger("cassandra.sai.max_string_term_size_kb", 1) * 1024;
-    public static final int MAX_FROZEN_TERM_SIZE = Integer.getInteger("cassandra.sai.max_frozen_term_size_kb", 5) * 1024;
+    public static final int MAX_STRING_TERM_SIZE = SAI_MAX_STRING_TERM_SIZE.getInt() * 1024;
+    public static final int MAX_FROZEN_TERM_SIZE = SAI_MAX_FROZEN_TERM_SIZE.getInt() * 1024;
     public static final String TERM_OVERSIZE_MESSAGE = "Can't add term of column {} to index for key: {}, term size {} " +
                                                        "max allowed size {}, use analyzed = true (if not yet set) for that column.";
 
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/sortedterms/SortedTermsWriter.java b/src/java/org/apache/cassandra/index/sai/disk/v1/sortedterms/SortedTermsWriter.java
index 9cbb232649..a3b810680e 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/sortedterms/SortedTermsWriter.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/sortedterms/SortedTermsWriter.java
@@ -195,7 +195,7 @@ public class SortedTermsWriter implements Closeable
 
     private void copyBytes(ByteComparable source, BytesRefBuilder dest)
     {
-        ByteSource byteSource = source.asComparableBytes(ByteComparable.Version.OSS42);
+        ByteSource byteSource = source.asComparableBytes(ByteComparable.Version.OSS50);
         int val;
         while ((val = byteSource.next()) != ByteSource.END_OF_STREAM)
             dest.append((byte) val);
diff --git a/src/java/org/apache/cassandra/index/sai/disk/v1/trie/TriePrefixSearcher.java b/src/java/org/apache/cassandra/index/sai/disk/v1/trie/TriePrefixSearcher.java
index 8acacd1aea..2fe6c46404 100644
--- a/src/java/org/apache/cassandra/index/sai/disk/v1/trie/TriePrefixSearcher.java
+++ b/src/java/org/apache/cassandra/index/sai/disk/v1/trie/TriePrefixSearcher.java
@@ -28,10 +28,10 @@ import org.apache.cassandra.utils.bytecomparable.ByteSource;
 
 /**
  * Thread-unsafe specialized value -> long trie prefix searcher.
- *
+ * <p>
  * This is a specialization of the {@link Walker} that will look for a prefix or
  * a complete term and return the first payload associated with the term.
- *
+ * <p>
  * TODO Make generic to handle any payload type.
  * TODO Extend search to return first payload or all sub-payloads as iterator (LIKE support?)
  */
diff --git a/src/java/org/apache/cassandra/index/sai/utils/TypeUtil.java b/src/java/org/apache/cassandra/index/sai/utils/TypeUtil.java
index e72b496395..220c24c337 100644
--- a/src/java/org/apache/cassandra/index/sai/utils/TypeUtil.java
+++ b/src/java/org/apache/cassandra/index/sai/utils/TypeUtil.java
@@ -118,7 +118,7 @@ public class TypeUtil
      */
     public static ByteComparable min(ByteComparable a, ByteComparable b)
     {
-        return a == null ? b : (b == null || ByteComparable.compare(b, a, ByteComparable.Version.OSS42) > 0) ? a : b;
+        return a == null ? b : (b == null || ByteComparable.compare(b, a, ByteComparable.Version.OSS50) > 0) ? a : b;
     }
 
     /**
@@ -127,7 +127,7 @@ public class TypeUtil
      */
     public static ByteComparable max(ByteComparable a, ByteComparable b)
     {
-        return a == null ? b : (b == null || ByteComparable.compare(b, a, ByteComparable.Version.OSS42) < 0) ? a : b;
+        return a == null ? b : (b == null || ByteComparable.compare(b, a, ByteComparable.Version.OSS50) < 0) ? a : b;
     }
 
     public static AbstractType<?> cellValueType(ColumnMetadata columnMetadata, IndexTarget.Type indexType)
@@ -474,7 +474,7 @@ public class TypeUtil
 
     public static ByteBuffer encodeDecimal(ByteBuffer value)
     {
-        ByteSource bs = DecimalType.instance.asComparableBytes(value, ByteComparable.Version.OSS42);
+        ByteSource bs = DecimalType.instance.asComparableBytes(value, ByteComparable.Version.OSS50);
         bs = ByteSource.cutOrRightPad(bs, DECIMAL_APPROXIMATION_BYTES, 0);
         return ByteBuffer.wrap(ByteSourceInverse.readBytes(bs, DECIMAL_APPROXIMATION_BYTES));
     }
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableWriter.java
index ea343fdd06..c5c175c802 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableWriter.java
@@ -18,6 +18,7 @@
 package org.apache.cassandra.io.sstable.format.bti;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
 
@@ -31,6 +32,7 @@ import org.slf4j.LoggerFactory;
 import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.DeletionTime;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.index.Index;
 import org.apache.cassandra.io.FSReadError;
 import org.apache.cassandra.io.FSWriteError;
 import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
@@ -379,9 +381,9 @@ public class BtiTableWriter extends SortedTableWriter<BtiFormatPartitionWriter>
         }
 
         @Override
-        public Builder addDefaultComponents()
+        public Builder addDefaultComponents(Collection<Index.Group> indexGroups)
         {
-            super.addDefaultComponents();
+            super.addDefaultComponents(indexGroups);
 
             addComponents(ImmutableSet.of(Components.PARTITION_INDEX, Components.ROW_INDEX));
 
diff --git a/src/java/org/apache/cassandra/io/tries/TrieNode.java b/src/java/org/apache/cassandra/io/tries/TrieNode.java
index 7da4ee39e0..02f7d15018 100644
--- a/src/java/org/apache/cassandra/io/tries/TrieNode.java
+++ b/src/java/org/apache/cassandra/io/tries/TrieNode.java
@@ -59,7 +59,7 @@ import org.apache.cassandra.io.util.SizedInts;
 public abstract class TrieNode
 {
     /** Value used to indicate a branch (e.g. for transition and lastTransition) does not exist. */
-    public static int NONE = -1;
+    public static final int NONE = -1;
 
     // Consumption (read) methods
 
diff --git a/src/java/org/apache/cassandra/io/tries/ValueIterator.java b/src/java/org/apache/cassandra/io/tries/ValueIterator.java
index f44e6e1580..6ddbebad2a 100644
--- a/src/java/org/apache/cassandra/io/tries/ValueIterator.java
+++ b/src/java/org/apache/cassandra/io/tries/ValueIterator.java
@@ -36,17 +36,17 @@ public class ValueIterator<CONCRETE extends ValueIterator<CONCRETE>> extends Wal
 {
     private final ByteSource limit;
     private final TransitionBytesCollector collector;
-    private IterationPosition stack;
+    protected IterationPosition stack;
     private long next;
 
-    static class IterationPosition
+    public static class IterationPosition
     {
         final long node;
         final int limit;
         final IterationPosition prev;
         int childIndex;
 
-        IterationPosition(long node, int childIndex, int limit, IterationPosition prev)
+        public IterationPosition(long node, int childIndex, int limit, IterationPosition prev)
         {
             super();
             this.node = node;
@@ -207,7 +207,7 @@ public class ValueIterator<CONCRETE extends ValueIterator<CONCRETE>> extends Wal
         return collector.toByteComparable();
     }
 
-    private long advanceNode()
+    protected long advanceNode()
     {
         long child;
         int transitionByte;
diff --git a/test/unit/org/apache/cassandra/index/sai/SAITester.java b/test/unit/org/apache/cassandra/index/sai/SAITester.java
index dc963153ae..9ec9ecb5b4 100644
--- a/test/unit/org/apache/cassandra/index/sai/SAITester.java
+++ b/test/unit/org/apache/cassandra/index/sai/SAITester.java
@@ -47,6 +47,7 @@ import javax.management.ObjectName;
 import com.google.common.collect.Sets;
 import org.junit.After;
 import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.rules.TestRule;
 import org.junit.rules.TestWatcher;
@@ -98,6 +99,7 @@ import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.Throwables;
 import org.apache.lucene.codecs.CodecUtil;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_RANDOM_SEED;
 import static org.apache.cassandra.inject.ActionBuilder.newActionBuilder;
 import static org.apache.cassandra.inject.Expression.quote;
 import static org.apache.cassandra.inject.InvokePointBuilder.newInvokePoint;
@@ -141,8 +143,11 @@ public abstract class SAITester extends CQLTester
 
     public static final PrimaryKey.Factory TEST_FACTORY = new PrimaryKey.Factory(EMPTY_COMPARATOR);
 
-    static
+    @BeforeClass
+    public static void setUpClass()
     {
+        CQLTester.setUpClass();
+
         // Ensure that the on-disk format statics are loaded before the test run
         Version.LATEST.onDiskFormat();
     }
@@ -556,7 +561,7 @@ public abstract class SAITester extends CQLTester
         return createTable(KEYSPACE, query);
     }
 
-    public UntypedResultSet execute(String query, Object... values) throws Throwable
+    public UntypedResultSet execute(String query, Object... values)
     {
         return executeFormattedQuery(formatQuery(query), values);
     }
@@ -776,13 +781,13 @@ public abstract class SAITester extends CQLTester
 
         Randomization()
         {
-            seed = Long.getLong("cassandra.test.random.seed", System.nanoTime());
+            seed = TEST_RANDOM_SEED.getLong(System.nanoTime());
             random = new Random(seed);
         }
 
         public void printSeedOnFailure()
         {
-            logger.error("Randomized test failed. To rerun test use -Dcassandra.test.random.seed=" + seed);
+            logger.error("Randomized test failed. To rerun test use -D{}={}", TEST_RANDOM_SEED.getKey(), seed);
         }
 
         public int nextInt()
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexSearcherTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexSearcherTest.java
index 02e0c5b0aa..485d2b095a 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexSearcherTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/InvertedIndexSearcherTest.java
@@ -213,6 +213,6 @@ public class InvertedIndexSearcherTest extends SAIRandomizedTester
 
     private ByteBuffer wrap(ByteComparable bc)
     {
-        return ByteBuffer.wrap(ByteSourceInverse.readBytes(bc.asComparableBytes(ByteComparable.Version.OSS42)));
+        return ByteBuffer.wrap(ByteSourceInverse.readBytes(bc.asComparableBytes(ByteComparable.Version.OSS50)));
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentFlushTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentFlushTest.java
index 751064a9d5..5778a2a201 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentFlushTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/SegmentFlushTest.java
@@ -208,7 +208,7 @@ public class SegmentFlushTest
         ByteComparable term = iterator.next();
         PostingList postings = iterator.postings();
 
-        assertEquals(0, ByteComparable.compare(term, ByteComparable.fixedLength(expectedTerm), ByteComparable.Version.OSS42));
+        assertEquals(0, ByteComparable.compare(term, ByteComparable.fixedLength(expectedTerm), ByteComparable.Version.OSS50));
         assertEquals(minSegmentRowId == maxSegmentRowId ? 1 : 2, postings.size());
     }
 
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsReaderTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsReaderTest.java
index 1566668dc4..f6c9a72e47 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsReaderTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsReaderTest.java
@@ -91,7 +91,7 @@ public class TermsReaderTest extends SAIRandomizedTester
             for (ByteComparable term = iterator.next(); term != null; term = iterator.next())
             {
                 final ByteComparable expected = termsEnum.get(i++).left;
-                assertEquals(0, ByteComparable.compare(expected, term, ByteComparable.Version.OSS42));
+                assertEquals(0, ByteComparable.compare(expected, term, ByteComparable.Version.OSS50));
             }
         }
     }
@@ -122,7 +122,7 @@ public class TermsReaderTest extends SAIRandomizedTester
         {
             for (Pair<ByteComparable, LongArrayList> pair : termsEnum)
             {
-                final byte[] bytes = ByteSourceInverse.readBytes(pair.left.asComparableBytes(ByteComparable.Version.OSS42));
+                final byte[] bytes = ByteSourceInverse.readBytes(pair.left.asComparableBytes(ByteComparable.Version.OSS50));
                 QueryEventListener.TrieIndexEventListener listener = mock(QueryEventListener.TrieIndexEventListener.class);
                 when(listener.postingListEventListener()).thenReturn(mock(QueryEventListener.PostingListEventListener.class));
                 try (PostingList actualPostingList = reader.exactMatch(ByteComparable.fixedLength(bytes),
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsScanner.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsScanner.java
index 2626d172d1..b317f4f301 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsScanner.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/TermsScanner.java
@@ -45,8 +45,8 @@ public class TermsScanner implements TermsIterator
     {
         this.postingsFile = postingsFile;
         this.iterator = new TrieTermsIterator(termFile.instantiateRebufferer(null), trieRoot);
-        this.minTerm = ByteBuffer.wrap(ByteSourceInverse.readBytes(iterator.getMinTerm().asComparableBytes(ByteComparable.Version.OSS42)));
-        this.maxTerm = ByteBuffer.wrap(ByteSourceInverse.readBytes(iterator.getMaxTerm().asComparableBytes(ByteComparable.Version.OSS42)));
+        this.minTerm = ByteBuffer.wrap(ByteSourceInverse.readBytes(iterator.getMinTerm().asComparableBytes(ByteComparable.Version.OSS50)));
+        this.maxTerm = ByteBuffer.wrap(ByteSourceInverse.readBytes(iterator.getMaxTerm().asComparableBytes(ByteComparable.Version.OSS50)));
     }
 
     @Override
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/sortedterms/SortedTermsTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/sortedterms/SortedTermsTest.java
index cc7608cbb7..9192665de6 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/sortedterms/SortedTermsTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/sortedterms/SortedTermsTest.java
@@ -75,13 +75,13 @@ public class SortedTermsTest extends SAIRandomizedTester
                                                                   trieWriter))
             {
                 ByteBuffer buffer = Int32Type.instance.decompose(99999);
-                ByteSource byteSource = Int32Type.instance.asComparableBytes(buffer, ByteComparable.Version.OSS42);
+                ByteSource byteSource = Int32Type.instance.asComparableBytes(buffer, ByteComparable.Version.OSS50);
                 byte[] bytes1 = ByteSourceInverse.readBytes(byteSource);
 
                 writer.add(ByteComparable.fixedLength(bytes1));
 
                 buffer = Int32Type.instance.decompose(444);
-                byteSource = Int32Type.instance.asComparableBytes(buffer, ByteComparable.Version.OSS42);
+                byteSource = Int32Type.instance.asComparableBytes(buffer, ByteComparable.Version.OSS50);
                 byte[] bytes2 = ByteSourceInverse.readBytes(byteSource);
 
                 assertThrows(IllegalArgumentException.class, () -> writer.add(ByteComparable.fixedLength(bytes2)));
@@ -154,7 +154,7 @@ public class SortedTermsTest extends SAIRandomizedTester
             {
                 ByteComparable term = cursor.term();
 
-                byte[] bytes = ByteSourceInverse.readBytes(term.asComparableBytes(ByteComparable.Version.OSS42));
+                byte[] bytes = ByteSourceInverse.readBytes(term.asComparableBytes(ByteComparable.Version.OSS50));
                 assertArrayEquals(terms.get(x), bytes);
                 x++;
             }
@@ -181,7 +181,7 @@ public class SortedTermsTest extends SAIRandomizedTester
             {
                 ByteComparable term = cursor.term();
 
-                byte[] bytes = ByteSourceInverse.readBytes(term.asComparableBytes(ByteComparable.Version.OSS42));
+                byte[] bytes = ByteSourceInverse.readBytes(term.asComparableBytes(ByteComparable.Version.OSS50));
                 assertArrayEquals(terms.get(x), bytes);
 
                 x++;
@@ -210,7 +210,7 @@ public class SortedTermsTest extends SAIRandomizedTester
                 cursor.seekToPointId(x);
                 ByteComparable term = cursor.term();
 
-                byte[] bytes = ByteSourceInverse.readBytes(term.asComparableBytes(ByteComparable.Version.OSS42));
+                byte[] bytes = ByteSourceInverse.readBytes(term.asComparableBytes(ByteComparable.Version.OSS50));
                 assertArrayEquals(terms.get(x), bytes);
             }
         });
@@ -223,7 +223,7 @@ public class SortedTermsTest extends SAIRandomizedTester
                 cursor.seekToPointId(x);
                 ByteComparable term = cursor.term();
 
-                byte[] bytes = ByteSourceInverse.readBytes(term.asComparableBytes(ByteComparable.Version.OSS42));
+                byte[] bytes = ByteSourceInverse.readBytes(term.asComparableBytes(ByteComparable.Version.OSS50));
                 assertArrayEquals(terms.get(x), bytes);
             }
         });
@@ -237,7 +237,7 @@ public class SortedTermsTest extends SAIRandomizedTester
                 cursor.seekToPointId(target);
                 ByteComparable term = cursor.term();
 
-                byte[] bytes = ByteSourceInverse.readBytes(term.asComparableBytes(ByteComparable.Version.OSS42));
+                byte[] bytes = ByteSourceInverse.readBytes(term.asComparableBytes(ByteComparable.Version.OSS50));
                 assertArrayEquals(terms.get(target), bytes);
             }
         });
@@ -276,7 +276,7 @@ public class SortedTermsTest extends SAIRandomizedTester
                 for (int x = 0; x < 1000 * 4; x++)
                 {
                     ByteBuffer buffer = Int32Type.instance.decompose(x);
-                    ByteSource byteSource = Int32Type.instance.asComparableBytes(buffer, ByteComparable.Version.OSS42);
+                    ByteSource byteSource = Int32Type.instance.asComparableBytes(buffer, ByteComparable.Version.OSS50);
                     byte[] bytes = ByteSourceInverse.readBytes(byteSource);
                     terms.add(bytes);
 
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/trie/TriePrefixSearcherTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/trie/TriePrefixSearcherTest.java
index 8f5ac200a6..5537f05e2d 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/trie/TriePrefixSearcherTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/trie/TriePrefixSearcherTest.java
@@ -52,9 +52,9 @@ public class TriePrefixSearcherTest extends SAIRandomizedTester
         try (FileHandle trieFileHandle = indexDescriptor.createPerSSTableFileHandle(IndexComponent.PRIMARY_KEY_TRIE);
              TriePrefixSearcher searcher = new TriePrefixSearcher(trieFileHandle.instantiateRebufferer(null), trieFilePointer))
         {
-            assertEquals(1L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString("abcdef"), ByteComparable.Version.OSS42)));
-            assertEquals(2L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString("abdefg"), ByteComparable.Version.OSS42)));
-            assertEquals(3L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString("abdfgh"), ByteComparable.Version.OSS42)));
+            assertEquals(1L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString("abcdef"), ByteComparable.Version.OSS50)));
+            assertEquals(2L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString("abdefg"), ByteComparable.Version.OSS50)));
+            assertEquals(3L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString("abdfgh"), ByteComparable.Version.OSS50)));
         }
     }
 
@@ -66,13 +66,13 @@ public class TriePrefixSearcherTest extends SAIRandomizedTester
         try (FileHandle trieFileHandle = indexDescriptor.createPerSSTableFileHandle(IndexComponent.PRIMARY_KEY_TRIE);
              TriePrefixSearcher searcher = new TriePrefixSearcher(trieFileHandle.instantiateRebufferer(null), trieFilePointer))
         {
-            assertEquals(1L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString(""), ByteComparable.Version.OSS42)));
-            assertEquals(1L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString("a"), ByteComparable.Version.OSS42)));
-            assertEquals(1L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString("ab"), ByteComparable.Version.OSS42)));
-            assertEquals(1L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString("abc"), ByteComparable.Version.OSS42)));
-            assertEquals(2L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString("abd"), ByteComparable.Version.OSS42)));
-            assertEquals(2L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString("abde"), ByteComparable.Version.OSS42)));
-            assertEquals(3L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString("abdf"), ByteComparable.Version.OSS42)));
+            assertEquals(1L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString(""), ByteComparable.Version.OSS50)));
+            assertEquals(1L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString("a"), ByteComparable.Version.OSS50)));
+            assertEquals(1L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString("ab"), ByteComparable.Version.OSS50)));
+            assertEquals(1L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString("abc"), ByteComparable.Version.OSS50)));
+            assertEquals(2L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString("abd"), ByteComparable.Version.OSS50)));
+            assertEquals(2L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString("abde"), ByteComparable.Version.OSS50)));
+            assertEquals(3L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString("abdf"), ByteComparable.Version.OSS50)));
         }
     }
 
@@ -84,7 +84,7 @@ public class TriePrefixSearcherTest extends SAIRandomizedTester
         try (FileHandle trieFileHandle = indexDescriptor.createPerSSTableFileHandle(IndexComponent.PRIMARY_KEY_TRIE);
              TriePrefixSearcher searcher = new TriePrefixSearcher(trieFileHandle.instantiateRebufferer(null), trieFilePointer))
         {
-            assertEquals(-1L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString("b"), ByteComparable.Version.OSS42)));
+            assertEquals(-1L, searcher.prefixSearch(UTF8Type.instance.asComparableBytes(UTF8Type.instance.fromString("b"), ByteComparable.Version.OSS50)));
         }
     }
 
@@ -96,10 +96,10 @@ public class TriePrefixSearcherTest extends SAIRandomizedTester
         try (FileHandle trieFileHandle = indexDescriptor.createPerSSTableFileHandle(IndexComponent.PRIMARY_KEY_TRIE);
              TriePrefixSearcher searcher = new TriePrefixSearcher(trieFileHandle.instantiateRebufferer(null), trieFilePointer))
         {
-            assertEquals(1L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS42, "abc", "def", "ghi")));
-            assertEquals(2L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS42, "abc", "def", "jkl")));
-            assertEquals(3L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS42, "abc", "ghi", "jkl")));
-            assertEquals(4L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS42, "def", "ghi", "jkl")));
+            assertEquals(1L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS50, "abc", "def", "ghi")));
+            assertEquals(2L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS50, "abc", "def", "jkl")));
+            assertEquals(3L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS50, "abc", "ghi", "jkl")));
+            assertEquals(4L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS50, "def", "ghi", "jkl")));
         }
     }
 
@@ -111,14 +111,14 @@ public class TriePrefixSearcherTest extends SAIRandomizedTester
         try (FileHandle trieFileHandle = indexDescriptor.createPerSSTableFileHandle(IndexComponent.PRIMARY_KEY_TRIE);
              TriePrefixSearcher searcher = new TriePrefixSearcher(trieFileHandle.instantiateRebufferer(null), trieFilePointer))
         {
-            assertEquals(1L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS42, "ab")));
-            assertEquals(1L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS42, "abc")));
-            assertEquals(1L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS42, "abc", "de")));
-            assertEquals(1L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS42, "abc", "def")));
-            assertEquals(1L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS42, "abc", "def", "gh")));
-            assertEquals(2L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS42, "abc", "def", "jk")));
-            assertEquals(3L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS42, "abc", "ghi")));
-            assertEquals(4L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS42, "d")));
+            assertEquals(1L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS50, "ab")));
+            assertEquals(1L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS50, "abc")));
+            assertEquals(1L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS50, "abc", "de")));
+            assertEquals(1L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS50, "abc", "def")));
+            assertEquals(1L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS50, "abc", "def", "gh")));
+            assertEquals(2L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS50, "abc", "def", "jk")));
+            assertEquals(3L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS50, "abc", "ghi")));
+            assertEquals(4L, searcher.prefixSearch(createMultiPart(ByteComparable.Version.OSS50, "d")));
         }
     }
 
diff --git a/test/unit/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryTest.java b/test/unit/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryTest.java
index 6b672032a9..273765c3e8 100644
--- a/test/unit/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/disk/v1/trie/TrieTermsDictionaryTest.java
@@ -39,7 +39,7 @@ import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
-import static org.apache.cassandra.utils.bytecomparable.ByteComparable.Version.OSS42;
+import static org.apache.cassandra.utils.bytecomparable.ByteComparable.Version.OSS50;
 import static org.apache.cassandra.utils.bytecomparable.ByteComparable.compare;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -114,7 +114,7 @@ public class TrieTermsDictionaryTest extends SAIRandomizedTester
                 assertTrue(expected.hasNext());
                 final Pair<ByteComparable, Long> actual = iterator.next();
 
-                assertEquals(0, compare(expected.next(), actual.left, OSS42));
+                assertEquals(0, compare(expected.next(), actual.left, OSS50));
                 assertEquals(offset++, actual.right.longValue());
             }
             assertFalse(expected.hasNext());
@@ -141,11 +141,11 @@ public class TrieTermsDictionaryTest extends SAIRandomizedTester
         {
             final ByteComparable expectedMaxTerm = byteComparables.get(byteComparables.size() - 1);
             final ByteComparable actualMaxTerm = reader.getMaxTerm();
-            assertEquals(0, compare(expectedMaxTerm, actualMaxTerm, OSS42));
+            assertEquals(0, compare(expectedMaxTerm, actualMaxTerm, OSS50));
 
             final ByteComparable expectedMinTerm = byteComparables.get(0);
             final ByteComparable actualMinTerm = reader.getMinTerm();
-            assertEquals(0, compare(expectedMinTerm, actualMinTerm, OSS42));
+            assertEquals(0, compare(expectedMinTerm, actualMinTerm, OSS50));
         }
     }
 
diff --git a/test/unit/org/apache/cassandra/index/sai/memory/RAMStringIndexerTest.java b/test/unit/org/apache/cassandra/index/sai/memory/RAMStringIndexerTest.java
index cc7424892f..56ea67a1d1 100644
--- a/test/unit/org/apache/cassandra/index/sai/memory/RAMStringIndexerTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/memory/RAMStringIndexerTest.java
@@ -95,7 +95,7 @@ public class RAMStringIndexerTest extends SAIRandomizedTester
         while (terms.hasNext())
         {
             term = terms.next();
-            final ByteBuffer decoded = ByteBuffer.wrap(ByteSourceInverse.readBytes(term.asComparableBytes(ByteComparable.Version.OSS42)));
+            final ByteBuffer decoded = ByteBuffer.wrap(ByteSourceInverse.readBytes(term.asComparableBytes(ByteComparable.Version.OSS50)));
             assertEquals(String.format("%04d", termOrd), string(decoded));
 
             try (PostingList postingList = terms.postings())
diff --git a/test/unit/org/apache/cassandra/index/sai/memory/TrieMemoryIndexTest.java b/test/unit/org/apache/cassandra/index/sai/memory/TrieMemoryIndexTest.java
index be5d3bae2d..9cf8ff5318 100644
--- a/test/unit/org/apache/cassandra/index/sai/memory/TrieMemoryIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/memory/TrieMemoryIndexTest.java
@@ -220,7 +220,7 @@ public class TrieMemoryIndexTest extends SAIRandomizedTester
                                                           ? ByteComparable.fixedLength(decompose.apply(rowId))
                                                           : version -> type.asComparableBytes(decompose.apply(rowId), version);
             final ByteComparable actualByteComparable = pair.left;
-            assertEquals("Mismatch at: " + i, 0, ByteComparable.compare(expectedByteComparable, actualByteComparable, ByteComparable.Version.OSS42));
+            assertEquals("Mismatch at: " + i, 0, ByteComparable.compare(expectedByteComparable, actualByteComparable, ByteComparable.Version.OSS50));
 
             i++;
         }
diff --git a/test/unit/org/apache/cassandra/index/sai/plan/OperationTest.java b/test/unit/org/apache/cassandra/index/sai/plan/OperationTest.java
index 10ed7f5035..e4da6c9fe3 100644
--- a/test/unit/org/apache/cassandra/index/sai/plan/OperationTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/plan/OperationTest.java
@@ -71,6 +71,7 @@ import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.FBUtilities;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_CONFIG;
 import static org.apache.cassandra.db.marshal.Int32Type.instance;
 
 public class OperationTest
@@ -91,7 +92,7 @@ public class OperationTest
     @BeforeClass
     public static void loadSchema() throws ConfigurationException
     {
-        System.setProperty("cassandra.config", "cassandra-murmur.yaml");
+        CASSANDRA_CONFIG.setString("cassandra-murmur.yaml");
 
         SchemaLoader.loadSchema();
 
diff --git a/test/unit/org/apache/cassandra/index/sai/utils/AbstractPrimaryKeyTester.java b/test/unit/org/apache/cassandra/index/sai/utils/AbstractPrimaryKeyTester.java
index 2897d9983d..77d334ccab 100644
--- a/test/unit/org/apache/cassandra/index/sai/utils/AbstractPrimaryKeyTester.java
+++ b/test/unit/org/apache/cassandra/index/sai/utils/AbstractPrimaryKeyTester.java
@@ -122,7 +122,7 @@ public class AbstractPrimaryKeyTester extends SAIRandomizedTester
     {
         assertEquals(expected, ByteComparable.compare(a::asComparableBytes,
                                                       b::asComparableBytes,
-                                                      ByteComparable.Version.OSS42));
+                                                      ByteComparable.Version.OSS50));
     }
 
     void assertCompareToAndEquals(PrimaryKey a, PrimaryKey b, int expected)
diff --git a/test/unit/org/apache/cassandra/index/sai/utils/TypeUtilTest.java b/test/unit/org/apache/cassandra/index/sai/utils/TypeUtilTest.java
index 642233ac57..9f91c271ba 100644
--- a/test/unit/org/apache/cassandra/index/sai/utils/TypeUtilTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/utils/TypeUtilTest.java
@@ -245,7 +245,7 @@ public class TypeUtilTest extends SAIRandomizedTester
             // simulate: saving into on-disk trie
             ByteComparable t0 = ByteComparable.fixedLength(b0);
             ByteComparable t1 = ByteComparable.fixedLength(b1);
-            assertTrue("#" + i, ByteComparable.compare(t0, t1, ByteComparable.Version.OSS42) <= 0);
+            assertTrue("#" + i, ByteComparable.compare(t0, t1, ByteComparable.Version.OSS50) <= 0);
         }
     }
 }
diff --git a/test/unit/org/apache/cassandra/index/sai/virtual/SegmentsSystemViewTest.java b/test/unit/org/apache/cassandra/index/sai/virtual/SegmentsSystemViewTest.java
index f37dcf86f0..910f9f11fa 100644
--- a/test/unit/org/apache/cassandra/index/sai/virtual/SegmentsSystemViewTest.java
+++ b/test/unit/org/apache/cassandra/index/sai/virtual/SegmentsSystemViewTest.java
@@ -51,6 +51,8 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertTrue;
 
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_ENCRYPTION;
+
 /**
  * Tests the virtual table exposing SSTable index segment metadata.
  */
@@ -170,7 +172,7 @@ public class SegmentsSystemViewTest extends SAITester
                     indexLengths.put(indexType, value + length);
                 }
             }
-            if (!Boolean.parseBoolean(System.getProperty("cassandra.test.encryption", "false")))
+            if (!TEST_ENCRYPTION.getBoolean())
                 assertEquals(indexFileLengths(), indexLengths);
         }
 
diff --git a/test/unit/org/apache/cassandra/io/tries/WalkerTest.java b/test/unit/org/apache/cassandra/io/tries/WalkerTest.java
index deb37e743a..d47bf4d2b9 100644
--- a/test/unit/org/apache/cassandra/io/tries/WalkerTest.java
+++ b/test/unit/org/apache/cassandra/io/tries/WalkerTest.java
@@ -307,5 +307,4 @@ public class WalkerTest extends AbstractTrieTestBase
         s = StringUtils.leftPad(s, size, '0');
         return source(s);
     }
-
 }
diff --git a/test/unit/org/apache/cassandra/io/util/SizedIntsTest.java b/test/unit/org/apache/cassandra/io/util/SizedIntsTest.java
index 9a3a953fcf..8793d46431 100644
--- a/test/unit/org/apache/cassandra/io/util/SizedIntsTest.java
+++ b/test/unit/org/apache/cassandra/io/util/SizedIntsTest.java
@@ -35,7 +35,6 @@ import static org.quicktheories.generators.SourceDSL.longs;
 
 public class SizedIntsTest
 {
-
     @Test
     public void nonZeroSize()
     {
diff --git a/test/unit/org/apache/cassandra/io/util/TailOverridingRebuffererTest.java b/test/unit/org/apache/cassandra/io/util/TailOverridingRebuffererTest.java
index 7bb65a04bd..85da3573c8 100644
--- a/test/unit/org/apache/cassandra/io/util/TailOverridingRebuffererTest.java
+++ b/test/unit/org/apache/cassandra/io/util/TailOverridingRebuffererTest.java
@@ -18,7 +18,6 @@
 
 package org.apache.cassandra.io.util;
 
-import java.io.IOException;
 import java.nio.ByteBuffer;
 
 import org.junit.Test;
@@ -32,7 +31,6 @@ import static org.mockito.Mockito.reset;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-
 public class TailOverridingRebuffererTest
 {
     ByteBuffer head = ByteBuffer.wrap(new byte[]{ 1, 2, 3, 4, 5, 6, 7, 8 });
@@ -101,7 +99,7 @@ public class TailOverridingRebuffererTest
     }
 
     @Test
-    public void testOtherMethods() throws IOException
+    public void testOtherMethods()
     {
         Rebufferer tor = new TailOverridingRebufferer(r, 8, tail.duplicate());
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org