You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by if...@apache.org on 2023/03/13 10:01:22 UTC

[cassandra-harry] 04/06: Implement concurrent quiescent checker

This is an automated email from the ASF dual-hosted git repository.

ifesdjeen pushed a commit to branch CASSANDRA-18318
in repository https://gitbox.apache.org/repos/asf/cassandra-harry.git

commit e465026ba243deb21e79b94de481973c53aad3b7
Author: Alex Petrov <al...@apple.com>
AuthorDate: Fri Mar 3 09:38:59 2023 +0100

    Implement concurrent quiescent checker
    
    Patch by Alex Petrov; reviewed by Caleb Rackliffe and Abe Ratnofsky for CASSANDRA-18315
---
 Makefile                                           |  18 +-
 README.md                                          | 104 ++++-
 bin/build-shaded-dtest-for-harry.sh                |  51 +++
 bin/relocate-dependencies-harry.pom                | 154 ++++++++
 conf/default.yaml                                  |   3 -
 conf/external.yaml                                 |  41 +-
 harry-core/pom.xml                                 |  10 +
 harry-core/src/harry/core/Configuration.java       | 178 ++++++---
 harry-core/src/harry/core/Run.java                 |   2 -
 .../src/harry/corruptor/AddExtraRowCorruptor.java  |  11 +-
 harry-core/src/harry/data/ResultSetRow.java        |  30 +-
 harry-core/src/harry/ddl/SchemaGenerators.java     |   5 +-
 harry-core/src/harry/dsl/HistoryBuilder.java       |   5 +
 .../harry/model/AlwaysSamePartitionSelector.java   |   5 +
 harry-core/src/harry/model/OpSelectors.java        |  26 +-
 .../src/harry/model/QueryingNoOpValidator.java     |  21 +-
 harry-core/src/harry/model/QuiescentChecker.java   |  41 +-
 .../model/QuiescentLocalStateCheckerBase.java      | 110 ++++++
 .../harry/model}/RepairingLocalStateValidator.java |  95 ++---
 harry-core/src/harry/model/clock/OffsetClock.java  |  17 +-
 .../src/harry/model/sut/TokenPlacementModel.java   | 218 +++++++++++
 .../sut/injvm}/InJVMTokenAwareVisitExecutor.java   |  85 +++-
 .../src/harry/model/sut/injvm/InJvmSut.java        |  30 +-
 .../src/harry/model/sut/injvm}/InJvmSutBase.java   |  74 ++--
 .../model/sut/injvm/MixedVersionInJvmSut.java      |  74 ++++
 .../sut/injvm/QuiescentLocalStateChecker.java      |  76 ++++
 harry-core/src/harry/reconciler/Reconciler.java    |  62 +--
 harry-core/src/harry/runner/DataTracker.java       |  69 ++--
 .../src/harry/runner/DefaultDataTracker.java       | 146 ++++---
 harry-core/src/harry/runner/HarryRunner.java       |  64 +---
 .../src/harry/runner/LockingDataTracker.java       | 257 +++++++++++++
 harry-core/src/harry/runner/Runner.java            | 426 ++++++++-------------
 harry-core/src/harry/runner/StagedRunner.java      | 133 +------
 .../src/harry/runner/TrivialShrinker.java          |   4 +-
 harry-core/src/harry/runner/UpToLtsRunner.java     |  46 ++-
 harry-core/src/harry/util/ByteUtils.java           |  47 +--
 .../src/harry/util/StringUtils.java                |  23 +-
 .../src/harry/visitors/AllPartitionsValidator.java | 146 +++----
 .../src/harry/visitors/CorruptingVisitor.java      |   1 +
 .../src/harry/visitors}/FaultInjectingVisitor.java |  48 ++-
 harry-core/src/harry/visitors/MutatingVisitor.java |  40 +-
 .../harry/visitors/RandomPartitionValidator.java   |  61 +++
 .../src/harry/visitors/SkippingVisitor.java        |   0
 harry-core/test/harry/operations/RelationTest.java |   5 +
 .../test/harry/runner/LockingDataTrackerTest.java  | 120 ++++++
 .../dependency-reduced-pom.xml                     |   2 +-
 harry-integration-external/pom.xml                 |   2 +-
 .../com/datastax/driver/core/UtilsAccessor.java    |  15 +-
 .../model/sut/external/ExternalClusterSut.java     |  56 ++-
 .../sut/external/QuiescentLocalStateChecker.java   |  80 ++++
 .../harry/runner/external/HarryRunnerExternal.java |   9 +-
 .../src/harry/runner/external/MiniStress.java      | 126 ++++++
 harry-integration/pom.xml                          |   4 +
 .../src/harry/model/sut/ByteUtils.java             | 115 ------
 .../InJvmSutConfiguration.java}                    |  53 +--
 .../MixedInJvmSutConfiguration.java}               |  66 +---
 harry-integration/src/harry/runner/Reproduce.java  |  52 ---
 .../generators/DataGeneratorsIntegrationTest.java  |  15 +-
 .../ConcurrentQuiescentCheckerIntegrationTest.java |  70 ++++
 .../harry/model/InJVMTokenAwareExecutorTest.java   |  32 +-
 .../test/harry/model/IntegrationTestBase.java      |   9 +-
 .../test/harry/model/ModelTestBase.java            |  93 +++--
 .../harry/model/QuerySelectorNegativeTest.java     |   1 +
 .../model/QuiescentCheckerIntegrationTest.java     |  36 +-
 .../QuiescentLocalStateCheckerIntegrationTest.java |  79 ++++
 pom.xml                                            |   2 +-
 66 files changed, 2697 insertions(+), 1402 deletions(-)

diff --git a/Makefile b/Makefile
index c43804c..31201df 100644
--- a/Makefile
+++ b/Makefile
@@ -14,13 +14,24 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
+CASSANDRA_SHA = 39ded1844f
 DOCKER_REPO   = org.apache.cassandra/harry/harry-runner
 
-mvn:
+cassandra:
+	git clone https://github.com/apache/cassandra.git
+
+cassandra-jar: cassandra
+	cp bin/build-shaded-dtest-for-harry.sh ./cassandra/
+	cp bin/relocate-dependencies-harry.pom ./cassandra/
+	cd cassandra; \
+		git checkout ${CASSANDRA_SHA}; \
+		./build-shaded-dtest-for-harry.sh
+
+package: cassandra-jar
 	rm -fr shared/*
 	mvn clean && mvn package -DskipTests
 
-img: mvn
+img: package
 	docker build -t ${DOCKER_REPO}:latest-local ./ -f docker/Dockerfile.local
 
 run: img
@@ -32,3 +43,6 @@ run-last:
 standalone:
 	rm -fr shared/*
 	mvn clean && mvn package -DskipTests -P standalone
+
+stress: package
+	@java -cp $(shell pwd)/harry-core/target/lib/*:$(shell pwd)/harry-integration/target/lib/*:$(shell pwd)/harry-integration-external/target/lib/*:$(shell pwd)/harry-core/target/*:$(shell pwd)/harry-integration/target/*:$(shell pwd)/harry-integration-external/target/* -ea harry.runner.external.MiniStress $(ARGS) 
diff --git a/README.md b/README.md
index 9c6b7bf..a09a345 100644
--- a/README.md
+++ b/README.md
@@ -4,6 +4,94 @@ The project aims to generate _reproducible_ workloads that are as close to real-
 as possible, while being able to _efficiently_ verify the cluster state against
 the model without pausing the workload itself.
 
+# TL;DR
+
+One way to see Harry is as a read/write stress tool that can check if the read results
+are consistent with what it knows that it has written. This means that in order to test
+Apache Cassandra with Harry, all you need is to run Harry.
+
+If you're using an external cluster (i.e. `./bin/cassandra -f`, CCM, docker, kubernetes,
+or just a deployed cluster), just use the command line mini stress tool:
+
+```
+make stress ARGS="conf/external.yaml --read=2 --write=2 --duration=60000 --validate-all-local"
+```
+
+This command will start perform concurrent read/write workload, 2 read and 2 write threads for
+60 seconds, and will validate presence of each written row on each node locally when it
+concludes. If it sees any inconsistencies, it will inform you accordingly. There are plenty
+configurables in the configuration files (schema, partition size, timestamp generation rules,
+authentication params, ratios for different mutation query types, and many more).
+
+An equivalent for an in-jvm cluster is:
+
+```
+try (Cluster cluster = builder().withNodes(3)
+                                .start())
+{
+    SchemaSpec schema = new SchemaSpec("harry", "test_table",
+                                       asList(pk("pk1", asciiType), pk("pk1", int64Type)),
+                                       asList(ck("ck1", asciiType), ck("ck1", int64Type)),
+                                       asList(regularColumn("regular1", asciiType), regularColumn("regular1", int64Type)),
+                                       asList(staticColumn("static1", asciiType), staticColumn("static1", int64Type)));
+
+    Configuration config = Configuration.fromFile("conf/example.yaml")
+                                        .unbuild()
+                                        .setKeyspaceDdl(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': %d};", schema.keyspace, 3))
+                                        .setSUT(() -> new InJvmSut(cluster))
+                                        .build();
+
+    Run run = config.createRun();
+
+    concurrent(run, config,
+               asList(pool("Writer", 2, MutatingVisitor::new),
+                      pool("Reader", 2, RandomPartitionValidator::new)),
+               2, TimeUnit.MINUTES)
+    .run();
+}
+```
+
+# What's available
+
+## System Under Test implementations
+
+ * `in_jvm/InJvmSut` - simple in-JVM-dtest system under test.
+ * `println/PrintlnSut` - system under test that prints to sdtout instead of executing queries on the cluster; useful for debugging.
+ * `mixed_in_jvm/MixedVersionInJvmSut` - in-JVM-dtest system under test that works with mixed version clusters.
+ * `external/ExternalClusterSut` - system under test that works with CCM, Docker, Kubernetes, or cluster you may. 
+   have deployed elsewhere 
+
+Both in-JVM SUTs have fault-injecting functionality available.
+
+## Visitors
+
+  * `single/SingleValidator` - visitor that runs several different read queries against a single partition that is associated with current logical timestamp, and validates their results using given model. 
+  * `all_partitions/AllPartitionsValidator` - concurrently validates all partitions that were visited during this run.
+  * `repair_and_validate_local_states/RepairingLocalStateValidator` - similar to `AllPartitionsValidator`, but performs repair before checking node states.
+  * `mutating/MutatingVisitor` - visitor that performs all sorts of mutations.
+  * `logging/LoggingVisitor` - similar to `MutatingVisitor`, but also logs all operations to a file; useful for debug purposes.
+  * `corrupting/CorruptingVisitor` - visitor that will deliberately change data in the partition it visits. Useful for negative tests (i.e. to ensure that your model actually detects data inconsistencies). 
+
+And more.
+
+## Models
+
+  * `querying_no_op/QueryingNoOpValidator` - a model that can be used to "simply" run random queries.
+  * `quiescent_checker/QuiescentChecker` - a model that can be used to verify results of any read that has no writes to the same partition_ concurrent to it. Should be used in conjunction with locking data tracker.
+  * `quiescent_local_state_checker/QuiescentLocalStateChecker` - a model that can check local states of each replica that has to own 
+
+## Runners 
+
+  * `sequential/SequentialRunner` - runs all visitors sequentially, in the loop, for a specified amount of time; useful for simple tests that do not have to exercise concurrent read/write path.   
+  * `concurrent/ConcurrentRunner` - runs all visitors concurrently, each visitor in its own thread, looped, for a specified amount of time; useful for things like concurrent read/write workloads.
+  * `chain/ChainRunner` - receives other runners as input, and runs them one after another once. Useful for both simple and complex scenarios that involve both read/write workloads, validating all partitions, exercising other node-local or cluster-wide operations.
+  * `staged/StagedRunner` - receives other runners (stages) as input, and runs them one after another in a loop; useful for implementing complex scenarios, such as read/write workloads followed by some cluster changing operations. 
+
+## Clock
+  
+  * `approximate_monotonic/ApproxomateMonotonicClock` - a timestamp supplier implementation that tries to keep as close to real time as possible, while preserving mapping from real-time to logical timestamps.
+  * `offset/OffsetClock` - a (monotonic) clock that supplies timestamps that do not have any relation to real time.
+
 # Introduction
 
 Harry has two primary modes of functionality:
@@ -653,17 +741,13 @@ of writing, there's no official repository where these jars are released, so you
 have to build it manually:
 
 ```
-git clone git@github.com:apache/cassandra.git
-cd cassandra
-./build-shaded-dtest-jar.sh 4.0-beta4 4.0.0-SNAPSHOT
-cd ~/../harry/
-mvn package -DskipTests -nsu
+make package
 mvn dependency:copy-dependencies
-java -cp harry-runner/target/harry-runner-0.0.1-SNAPSHOT.jar:$(find harrry-runner/target/dependency/*.jar | tr -s '\n' ':'). harry.runner.HarryRunner
-```
+java -cp harry-core/target/harry-core-0.0.2-SNAPSHOT.jar:$(find harrry-core/target/dependency/*.jar | tr -s '\n' ':'). harry.runner.HarryRunner
+
 
-`4.0-beta3` is a version of Cassandra which you can find in `build.xml`, and
-`4.0.0-SNAPSHOT` is a version of dtest jar that'll be installed under
+`4.2` is a version of Cassandra which you can find in `build.xml`, and
+`4.2-SNAPSHOT` is a version of dtest jar that'll be installed under
 `org.apache.cassandra:cassandra-dtest-local` in your `~/.m2/repository`.
 
 Alternatively, you can use a Docker container. For that just run:
@@ -671,7 +755,7 @@ Alternatively, you can use a Docker container. For that just run:
 ```
 git clone git@github.com:apache/cassandra.git
 cd cassandra
-./build-shaded-dtest-jar.sh 4.0-beta4 4.0.0-SNAPSHOT
+./build-shaded-dtest-jar.sh 4.2 4.2-SNAPSHOT
 cd ~/../harry/
 make run
 ```
diff --git a/bin/build-shaded-dtest-for-harry.sh b/bin/build-shaded-dtest-for-harry.sh
new file mode 100755
index 0000000..e384556
--- /dev/null
+++ b/bin/build-shaded-dtest-for-harry.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+
+export REVISION=`git rev-parse --short HEAD`
+export BASE_VERSION=$(cat build.xml | grep 'property name="base.version"' | awk -F "\"" '{print $4}')
+export VERSION=$BASE_VERSION
+export ARTIFACT_NAME=cassandra-dtest
+export REPO_DIR=~/.m2/repository/
+
+ant clean
+ant dtest-jar
+
+# Install the version that will be shaded
+mvn install:install-file                    \
+   -Dfile=./build/dtest-${BASE_VERSION}.jar \
+   -DgroupId=org.apache.cassandra           \
+   -DartifactId=${ARTIFACT_NAME}-local      \
+   -Dversion=${VERSION}-${REVISION}         \
+   -Dpackaging=jar                          \
+   -DgeneratePom=true
+
+# Create shaded artifact
+mvn -f relocate-dependencies-harry.pom versions:set -DnewVersion=${VERSION}-${REVISION} -U
+mvn -f relocate-dependencies-harry.pom package -DskipTests -nsu -Ddtest-local.version=${VERSION}-${REVISION} -U
+
+# Deploy shaded artifact
+mvn install:install-file                                 \
+   -Dfile=./target/${ARTIFACT_NAME}-shaded-${VERSION}-${REVISION}.jar \
+   -DgroupId=org.apache.cassandra                        \
+   -DartifactId=${ARTIFACT_NAME}-shaded                  \
+   -Dversion=${VERSION}-${REVISION}                      \
+   -Dpackaging=jar                                       \
+   -DgeneratePom=true                                    \
+   -DlocalRepositoryPath=${REPO_DIR}
diff --git a/bin/relocate-dependencies-harry.pom b/bin/relocate-dependencies-harry.pom
new file mode 100644
index 0000000..7d0aa3c
--- /dev/null
+++ b/bin/relocate-dependencies-harry.pom
@@ -0,0 +1,154 @@
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.apache.cassandra</groupId>
+    <artifactId>cassandra-dtest-shaded</artifactId>
+    <version>VERSION_PLACEHOLDER</version>
+    <packaging>jar</packaging>
+
+    <name>Cassandra in-jvm dtests shaded jar</name>
+
+    <properties>
+        <project.type>library</project.type>
+        <java.version>1.8</java.version>
+        <test.source.directory>src/test/unit/java</test.source.directory>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+
+        <junit.version>4.12</junit.version>
+        <maven.compiler.source>${java.version}</maven.compiler.source>
+        <maven.compiler.target>${java.version}</maven.compiler.target>
+        <dtest-local.version>DTEST_VERSION_PLACEHOLDER</dtest-local.version>
+    </properties>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>org.apache.cassandra</groupId>
+            <artifactId>cassandra-dtest-local</artifactId>
+            <version>${dtest-local.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>${junit.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <testSourceDirectory>${test.source.directory}</testSourceDirectory>
+
+        <plugins>
+            <!-- generate a shaded JAR -->
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.2.4</version>
+
+                <configuration>
+                    <createSourcesJar>false</createSourcesJar>
+
+                    <artifactSet>
+                        <includes>
+                            <include>org.apache.cassandra:cassandra-dtest-local</include>
+                        </includes>
+                    </artifactSet>
+
+                    <relocations>
+                        <relocation>
+                            <pattern>io.netty</pattern>
+                            <shadedPattern>relocated.shaded.io.netty</shadedPattern>
+                        </relocation>
+
+                        <relocation>
+                            <pattern>com.google</pattern>
+                            <shadedPattern>relocated.shaded.com.google</shadedPattern>
+                        </relocation>
+
+                        <relocation>
+                            <pattern>com.datastax</pattern>
+                            <shadedPattern>relocated.shaded.com.datastax</shadedPattern>
+                        </relocation>
+                    </relocations>
+
+                    <filters>
+                        <filter>
+                            <artifact>*:*</artifact>
+                            <excludes>
+                                <exclude>**/Log4j2Plugins.dat</exclude>
+                                <exclude>net/openhft/chronicle/wire/YamlWire$TextValueIn.class</exclude>
+                            </excludes>
+                        </filter>
+
+                        <filter>
+                            <artifact>io.netty:netty-*</artifact>
+                            <excludes>
+                                <exclude>META-INF/maven/</exclude>
+                                <exclude>META-INF/io.netty.versions.properties</exclude>
+                            </excludes>
+                        </filter>
+
+                        <filter>
+                            <artifact>com.google.guava:guava</artifact>
+                            <excludes>
+                                <exclude>META-INF/maven/</exclude>
+                            </excludes>
+                        </filter>
+
+                        <filter>
+                            <artifact>com.google.guava:failureaccess</artifact>
+                            <excludes>
+                                <exclude>META-INF/maven/</exclude>
+                            </excludes>
+                        </filter>
+
+                        <filter>
+                            <artifact>com.datastax.cassandra:cassandra-driver-core</artifact>
+                            <excludes>
+                                <exclude>META-INF/maven/</exclude>
+                            </excludes>
+                        </filter>
+
+                    </filters>
+                </configuration>
+
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+
+                        </configuration>
+                    </execution>
+                </executions>
+
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/conf/default.yaml b/conf/default.yaml
index c8d00b3..a004747 100644
--- a/conf/default.yaml
+++ b/conf/default.yaml
@@ -35,9 +35,6 @@ clock:
     epoch_length: 1
     epoch_time_unit: "SECONDS"
 
-run_time: 2
-run_time_unit: "HOURS"
-
 # System under test: a Cassandra node or cluster. Default implementation is in_jvm (in-jvm DTest cluster).
 # Harry also supports external clusters.
 system_under_test:
diff --git a/conf/external.yaml b/conf/external.yaml
index f6be4c7..b75547e 100644
--- a/conf/external.yaml
+++ b/conf/external.yaml
@@ -18,11 +18,25 @@ seed: 1596731732524
 
 # Default schema provider generates random schema
 schema_provider:
-  default: {}
-
-drop_schema: false
-create_schema: true
-truncate_table: false
+  fixed:
+    keyspace: harry
+    table: test_table
+    partition_keys:
+      pk1: bigint
+      pk2: ascii
+    clustering_keys:
+      ck1: ascii
+      ck2: bigint
+    regular_columns:
+      v1: ascii
+      v2: bigint
+      v3: ascii
+      v4: bigint
+    static_keys:
+      s1: ascii
+      s2: bigint
+      s3: ascii
+      s4: bigint
 
 # Clock is a component responsible for mapping _logical_ timestamps to _real-time_ ones.
 #
@@ -35,6 +49,10 @@ clock:
     epoch_length: 1
     epoch_time_unit: "SECONDS"
 
+drop_schema: false
+create_schema: true
+truncate_table: true
+
 # System under test: a Cassandra node or cluster. Default implementation is in_jvm (in-jvm DTest cluster).
 # Harry also supports external clusters.
 system_under_test:
@@ -44,12 +62,6 @@ system_under_test:
     username: null
     password: null
 
-# Model is responsible for tracking logical timestamps that
-# model:
-#  exhaustive_checker:
-#    max_seen_lts: 19
-#    max_complete_lts: 16
-
 # Partition descriptor selector controls how partitions is selected based on the current logical
 # timestamp. Default implementation is a sliding window of partition descriptors that will visit
 # one partition after the other in the window `slide_after_repeats` times. After that will
@@ -67,7 +79,7 @@ clustering_descriptor_selector:
   default:
     modifications_per_lts:
       type: "constant"
-      constant: 4
+      constant: 2
     rows_per_modification:
       type: "constant"
       constant: 2
@@ -114,3 +126,8 @@ runner:
 
 metric_reporter:
   no_op: {}
+
+data_tracker:
+  locking:
+    max_seen_lts: -1
+    max_complete_lts: -1
diff --git a/harry-core/pom.xml b/harry-core/pom.xml
index 153251c..a65fe5e 100755
--- a/harry-core/pom.xml
+++ b/harry-core/pom.xml
@@ -32,6 +32,16 @@
     <name>Harry Core</name>
 
     <dependencies>
+         <dependency>
+             <groupId>org.apache.cassandra</groupId>
+             <artifactId>dtest-api</artifactId>
+         </dependency>
+
+         <dependency>
+             <groupId>org.reflections</groupId>
+             <artifactId>reflections</artifactId>
+         </dependency>
+
          <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
diff --git a/harry-core/src/harry/core/Configuration.java b/harry-core/src/harry/core/Configuration.java
index 7cd8f72..88474b9 100644
--- a/harry-core/src/harry/core/Configuration.java
+++ b/harry-core/src/harry/core/Configuration.java
@@ -18,14 +18,23 @@
 
 package harry.core;
 
+import java.io.BufferedWriter;
 import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.lang.annotation.Annotation;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
+import java.util.stream.Collectors;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -38,14 +47,12 @@ import harry.ddl.SchemaGenerators;
 import harry.ddl.SchemaSpec;
 import harry.generators.Surjections;
 import harry.generators.distribution.Distribution;
-import harry.model.AlwaysSamePartitionSelector;
 import harry.model.Model;
 import harry.model.OpSelectors;
 import harry.model.QuiescentChecker;
 import harry.model.clock.ApproximateMonotonicClock;
-import harry.model.clock.OffsetClock;
-import harry.model.sut.PrintlnSut;
 import harry.model.sut.SystemUnderTest;
+import harry.runner.LockingDataTracker;
 import harry.visitors.AllPartitionsValidator;
 import harry.visitors.CorruptingVisitor;
 import harry.runner.DataTracker;
@@ -54,55 +61,42 @@ import harry.visitors.LoggingVisitor;
 import harry.visitors.MutatingVisitor;
 import harry.visitors.MutatingRowVisitor;
 import harry.visitors.OperationExecutor;
-import harry.visitors.ParallelRecentValidator;
 import harry.visitors.Visitor;
 import harry.visitors.RecentValidator;
 import harry.runner.Runner;
-import harry.visitors.Sampler;
 import harry.util.BitSet;
+import org.reflections.Reflections;
+import org.reflections.scanners.Scanners;
+import org.reflections.util.NameHelper;
 
 public class Configuration
 {
+    private enum NameUtils implements NameHelper
+    {
+        INSTANCE;
+    }
+
     private static final ObjectMapper mapper;
 
-    private static final String DEFAULT_KEYSPACE = "harry";
+    private static <A extends Annotation> Set<Class<?>> findClassesMarkedWith(Class<A> annotation)
+    {
+        Reflections reflections = new Reflections(org.reflections.util.ConfigurationBuilder.build("harry").setExpandSuperTypes(false));
+        Collection<Class<?>> classes = NameUtils.INSTANCE.forNames(reflections.get(Scanners.TypesAnnotated.get(annotation.getName())),
+                                                                   reflections.getConfiguration().getClassLoaders());
+        return new HashSet<>(classes);
+    }
 
     static
     {
+
         mapper = new ObjectMapper(new YAMLFactory()
                                   .disable(YAMLGenerator.Feature.USE_NATIVE_TYPE_ID)
                                   .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER)
                                   .disable(YAMLGenerator.Feature.CANONICAL_OUTPUT)
                                   .enable(YAMLGenerator.Feature.INDENT_ARRAYS));
-        mapper.registerSubtypes(Configuration.DebugApproximateMonotonicClockConfiguration.class);
-        mapper.registerSubtypes(Configuration.ApproximateMonotonicClockConfiguration.class);
-        mapper.registerSubtypes(Configuration.ConcurrentRunnerConfig.class);
-        mapper.registerSubtypes(Configuration.SequentialRunnerConfig.class);
-        mapper.registerSubtypes(Configuration.SingleVisitRunnerConfig.class);
-        mapper.registerSubtypes(Configuration.DefaultDataTrackerConfiguration.class);
-        mapper.registerSubtypes(Configuration.NoOpDataTrackerConfiguration.class);
-
-        mapper.registerSubtypes(Configuration.QuiescentCheckerConfig.class);
-        mapper.registerSubtypes(NoOpCheckerConfig.class);
-        mapper.registerSubtypes(Configuration.DefaultCDSelectorConfiguration.class);
-        mapper.registerSubtypes(Configuration.DefaultPDSelectorConfiguration.class);
-        mapper.registerSubtypes(Configuration.ConstantDistributionConfig.class);
-        mapper.registerSubtypes(DefaultSchemaProviderConfiguration.class);
-        mapper.registerSubtypes(MutatingRowVisitorConfiguration.class);
-
-        mapper.registerSubtypes(MutatingVisitorConfiguation.class);
-        mapper.registerSubtypes(LoggingVisitorConfiguration.class);
-        mapper.registerSubtypes(AllPartitionsValidatorConfiguration.class);
-        mapper.registerSubtypes(ParallelRecentValidator.ParallelRecentValidatorConfig.class);
-        mapper.registerSubtypes(Sampler.SamplerConfiguration.class);
-        mapper.registerSubtypes(CorruptingVisitorConfiguration.class);
-        mapper.registerSubtypes(RecentPartitionsValidatorConfiguration.class);
-        mapper.registerSubtypes(FixedSchemaProviderConfiguration.class);
-        mapper.registerSubtypes(AlwaysSamePartitionSelector.AlwaysSamePartitionSelectorConfiguration.class);
-        mapper.registerSubtypes(OffsetClock.OffsetClockConfiguration.class);
-        mapper.registerSubtypes(PrintlnSut.PrintlnSutConfiguration.class);
-        mapper.registerSubtypes(NoOpDataTrackerConfiguration.class);
-        mapper.registerSubtypes(NoOpMetricReporterConfiguration.class);
+
+        findClassesMarkedWith(JsonTypeName.class)
+        .forEach(mapper::registerSubtypes);
     }
 
     public final long seed;
@@ -156,6 +150,18 @@ public class Configuration
         mapper.registerSubtypes(classes);
     }
 
+    public static void toFile(File file, Configuration config)
+    {
+        try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file))))
+        {
+            bw.write(Configuration.toYamlString(config));
+            bw.flush();
+        }
+        catch (IOException e)
+        {
+            throw new RuntimeException(e);
+        }
+    }
     public static String toYamlString(Configuration config)
     {
         try
@@ -229,21 +235,21 @@ public class Configuration
 
             long seed = snapshot.seed;
 
-            DataTracker tracker = snapshot.data_tracker == null ? new DefaultDataTrackerConfiguration().make() : snapshot.data_tracker.make();
             OpSelectors.Rng rng = new OpSelectors.PCGFast(seed);
 
-            OpSelectors.MonotonicClock clock = snapshot.clock.make();
-
-            MetricReporter metricReporter = snapshot.metric_reporter.make();
-
             // TODO: validate that operation kind is compatible with schema, due to statics etc
             sut = snapshot.system_under_test.make();
-
             SchemaSpec schemaSpec = snapshot.schema_provider.make(seed, sut);
             schemaSpec.validate();
 
             OpSelectors.PdSelector pdSelector = snapshot.partition_descriptor_selector.make(rng);
+            DataTrackerConfiguration dataTrackerConfiguration = snapshot.data_tracker == null ? new DefaultDataTrackerConfiguration() : snapshot.data_tracker;
+            DataTracker tracker = dataTrackerConfiguration.make(pdSelector, schemaSpec);
+
             OpSelectors.DescriptorSelector descriptorSelector = snapshot.clustering_descriptor_selector.make(rng, schemaSpec);
+            OpSelectors.MonotonicClock clock = snapshot.clock.make();
+
+            MetricReporter metricReporter = snapshot.metric_reporter.make();
 
             return new Run(rng,
                            clock,
@@ -306,7 +312,7 @@ public class Configuration
             return this;
         }
 
-        public ConfigurationBuilder setDataTracker(String keyspace_ddl)
+        public ConfigurationBuilder setKeyspaceDdl(String keyspace_ddl)
         {
             this.keyspace_ddl = keyspace_ddl;
             return this;
@@ -396,14 +402,18 @@ public class Configuration
     {
         ConfigurationBuilder builder = new ConfigurationBuilder();
         builder.seed = seed;
+
         builder.schema_provider = schema_provider;
+        builder.keyspace_ddl = keyspace_ddl;
         builder.drop_schema = drop_schema;
         builder.create_schema = create_schema;
         builder.truncate_table = truncate_table;
 
+        builder.data_tracker = data_tracker;
         builder.clock = clock;
         builder.runner = runner;
         builder.system_under_test = system_under_test;
+        builder.metric_reporter = metric_reporter;
 
         builder.partition_descriptor_selector = partition_descriptor_selector;
         builder.clustering_descriptor_selector = clustering_descriptor_selector;
@@ -426,7 +436,7 @@ public class Configuration
         {
         }
 
-        public DataTracker make()
+        public DataTracker make(OpSelectors.PdSelector pdSelector, SchemaSpec schemaSpec)
         {
             return DataTracker.NO_OP;
         }
@@ -437,28 +447,60 @@ public class Configuration
     {
         public final long max_seen_lts;
         public final long max_complete_lts;
+        public final List<Long> reorder_buffer;
 
         public DefaultDataTrackerConfiguration()
         {
-            this(-1, -1);
+            this(-1, -1, null);
         }
 
         @JsonCreator
         public DefaultDataTrackerConfiguration(@JsonProperty(value = "max_seen_lts", defaultValue = "-1") long max_seen_lts,
-                                               @JsonProperty(value = "max_complete_lts", defaultValue = "-1") long max_complete_lts)
+                                               @JsonProperty(value = "max_complete_lts", defaultValue = "-1") long max_complete_lts,
+                                               @JsonProperty(value = "reorder_buffer", defaultValue = "null") List<Long> reorder_buffer)
         {
             this.max_seen_lts = max_seen_lts;
             this.max_complete_lts = max_complete_lts;
+            this.reorder_buffer = reorder_buffer;
         }
 
-        public DataTracker make()
+        @Override
+        public DataTracker make(OpSelectors.PdSelector pdSelector, SchemaSpec schemaSpec)
         {
-            DefaultDataTracker defaultDataTracker = new DefaultDataTracker();
-            defaultDataTracker.forceLts(max_seen_lts, max_complete_lts);
-            return defaultDataTracker;
+            DefaultDataTracker tracker = new DefaultDataTracker();
+            tracker.forceLts(max_seen_lts, max_complete_lts, reorder_buffer);
+            return tracker;
         }
     }
 
+    @JsonTypeName("locking")
+    public static class LockingDataTrackerConfiguration implements DataTrackerConfiguration
+    {
+        public final long max_seen_lts;
+        public final long max_complete_lts;
+        public final List<Long> reorder_buffer;
+
+        @JsonCreator
+        public LockingDataTrackerConfiguration(@JsonProperty(value = "max_seen_lts", defaultValue = "-1") long max_seen_lts,
+                                               @JsonProperty(value = "max_complete_lts", defaultValue = "-1") long max_complete_lts,
+                                               @JsonProperty(value = "reorder_buffer", defaultValue = "null") List<Long> reorder_buffer)
+        {
+            this.max_seen_lts = max_seen_lts;
+            this.max_complete_lts = max_complete_lts;
+            this.reorder_buffer = reorder_buffer;
+        }
+
+        @Override
+        public DataTracker make(OpSelectors.PdSelector pdSelector, SchemaSpec schemaSpec)
+        {
+            LockingDataTracker tracker = new LockingDataTracker(pdSelector, schemaSpec);
+            tracker.forceLts(max_seen_lts, max_complete_lts, reorder_buffer);
+            return tracker;
+        }
+    }
+
+
+
     @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT)
     public interface ClockConfiguration extends OpSelectors.MonotonicClockFactory
     {
@@ -538,22 +580,18 @@ public class Configuration
     @JsonTypeName("concurrent")
     public static class ConcurrentRunnerConfig implements RunnerConfiguration
     {
-        public final int concurrency;
-
-        @JsonProperty(value = "visitors")
-        public final List<VisitorConfiguration> visitorFactories;
+        @JsonProperty(value = "visitor_pools")
+        public final List<VisitorPoolConfiguration> visitor_pools;
 
         public final long run_time;
         public final TimeUnit run_time_unit;
 
         @JsonCreator
-        public ConcurrentRunnerConfig(@JsonProperty(value = "concurrency", defaultValue = "4") int concurrency,
-                                      @JsonProperty(value = "visitors") List<VisitorConfiguration> visitors,
+        public ConcurrentRunnerConfig(@JsonProperty(value = "visitor_pools") List<VisitorPoolConfiguration> visitor_pools,
                                       @JsonProperty(value = "run_time", defaultValue = "2") long runtime,
                                       @JsonProperty(value = "run_time_unit", defaultValue = "HOURS") TimeUnit runtimeUnit)
         {
-            this.concurrency = concurrency;
-            this.visitorFactories = visitors;
+            this.visitor_pools = visitor_pools;
             this.run_time = runtime;
             this.run_time_unit = runtimeUnit;
         }
@@ -561,7 +599,29 @@ public class Configuration
         @Override
         public Runner make(Run run, Configuration config)
         {
-            return new Runner.ConcurrentRunner(run, config, concurrency, visitorFactories, run_time, run_time_unit);
+            return new Runner.ConcurrentRunner(run, config, visitor_pools, run_time, run_time_unit);
+        }
+    }
+
+    public static class VisitorPoolConfiguration
+    {
+        public final String prefix;
+        public final int concurrency;
+        public final VisitorConfiguration visitor;
+
+        @JsonCreator
+        public VisitorPoolConfiguration(@JsonProperty(value = "prefix") String prefix,
+                                        @JsonProperty(value = "concurrency") int concurrency,
+                                        @JsonProperty(value = "visitor") VisitorConfiguration visitor)
+        {
+            this.prefix = prefix;
+            this.concurrency = concurrency;
+            this.visitor = visitor;
+        }
+
+        public static VisitorPoolConfiguration pool(String prefix, int concurrency, VisitorConfiguration visitor)
+        {
+            return new VisitorPoolConfiguration(prefix, concurrency, visitor);
         }
     }
 
@@ -998,7 +1058,7 @@ public class Configuration
 
         public Visitor make(Run run)
         {
-            return new AllPartitionsValidator(concurrency, trigger_after, run, modelConfiguration);
+            return new AllPartitionsValidator(run, concurrency, trigger_after, modelConfiguration);
         }
     }
 
diff --git a/harry-core/src/harry/core/Run.java b/harry-core/src/harry/core/Run.java
index e3bc203..be47d45 100644
--- a/harry-core/src/harry/core/Run.java
+++ b/harry-core/src/harry/core/Run.java
@@ -38,7 +38,6 @@ public class Run
 
     public final MetricReporter metricReporter;
 
-
     public Run(OpSelectors.Rng rng,
                OpSelectors.MonotonicClock clock,
                OpSelectors.PdSelector pdSelector,
@@ -63,7 +62,6 @@ public class Run
                 SystemUnderTest sut,
                 MetricReporter metricReporter)
     {
-
         this.rng = rng;
         this.clock = clock;
         this.pdSelector = pdSelector;
diff --git a/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java b/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java
index ace55b4..049c575 100644
--- a/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java
+++ b/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java
@@ -31,6 +31,7 @@ import harry.model.SelectHelper;
 import harry.model.sut.SystemUnderTest;
 import harry.operations.WriteHelper;
 import harry.operations.Query;
+import harry.runner.DataTracker;
 
 public class AddExtraRowCorruptor implements QueryResponseCorruptor
 {
@@ -38,34 +39,34 @@ public class AddExtraRowCorruptor implements QueryResponseCorruptor
 
     private final SchemaSpec schema;
     private final OpSelectors.MonotonicClock clock;
+    private final DataTracker tracker;
     private final OpSelectors.DescriptorSelector descriptorSelector;
 
     public AddExtraRowCorruptor(SchemaSpec schema,
                                 OpSelectors.MonotonicClock clock,
+                                DataTracker tracker,
                                 OpSelectors.DescriptorSelector descriptorSelector)
     {
         this.schema = schema;
         this.clock = clock;
+        this.tracker = tracker;
         this.descriptorSelector = descriptorSelector;
     }
 
     public boolean maybeCorrupt(Query query, SystemUnderTest sut)
     {
         Set<Long> cds = new HashSet<>();
-        long maxLts = 0;
+        long maxLts = tracker.maxStarted();
         for (Object[] obj : sut.execute(query.toSelectStatement(), SystemUnderTest.ConsistencyLevel.ALL))
         {
             ResultSetRow row = SelectHelper.resultSetToRow(schema, clock, obj);
             // TODO: extract CD cheaper
             cds.add(row.cd);
-            for (int i = 0; i < row.lts.length; i++)
-                maxLts = Math.max(maxLts, row.lts[i]);
         }
-
         boolean partitionIsFull = cds.size() >= descriptorSelector.maxPartitionSize();
 
         long attempt = 0;
-        long cd = descriptorSelector.randomCd(query.pd, attempt, schema);;
+        long cd = descriptorSelector.randomCd(query.pd, attempt, schema);
         while (!query.match(cd) || cds.contains(cd))
         {
             if (partitionIsFull)
diff --git a/harry-core/src/harry/data/ResultSetRow.java b/harry-core/src/harry/data/ResultSetRow.java
index 48cfc03..d6ff77b 100644
--- a/harry-core/src/harry/data/ResultSetRow.java
+++ b/harry-core/src/harry/data/ResultSetRow.java
@@ -21,6 +21,7 @@ package harry.data;
 import java.util.Arrays;
 
 import harry.ddl.SchemaSpec;
+import harry.util.StringUtils;
 
 public class ResultSetRow
 {
@@ -52,35 +53,22 @@ public class ResultSetRow
         return "resultSetRow("
                + pd +
                "L, " + cd +
-               (sds == null ? "" : "L, values(" + toString(sds) + ")") +
-               (slts == null ? "" : ", lts(" + toString(slts) + ")") +
-               ", values(" + toString(vds) + ")" +
-               ", lts(" + toString(lts) + ")" +
+               (sds == null ? "" : "L, values(" + StringUtils.toString(sds) + ")") +
+               (slts == null ? "" : ", lts(" + StringUtils.toString(slts) + ")") +
+               ", values(" + StringUtils.toString(vds) + ")" +
+               ", lts(" + StringUtils.toString(lts) + ")" +
                ")";
     }
 
-    public String toString(long[] arr)
-    {
-        String s = "";
-        for (int i = 0; i < arr.length; i++)
-        {
-            s += arr[i];
-            s += "L";
-            if (i < (arr.length - 1))
-                s += ',';
-        }
-        return s;
-    }
-
     public String toString(SchemaSpec schema)
     {
         return "resultSetRow("
                + pd +
                "L, " + cd +
-               (sds == null ? "" : "L, staticValues(" + toString(sds) + ")") +
-               (slts == null ? "" : ", slts(" + toString(slts) + ")") +
-               ", values(" + toString(vds) + ")" +
-               ", lts(" + toString(lts) + ")" +
+               (sds == null ? "" : "L, staticValues(" + StringUtils.toString(sds) + ")") +
+               (slts == null ? "" : ", slts(" + StringUtils.toString(slts) + ")") +
+               ", values(" + StringUtils.toString(vds) + ")" +
+               ", lts(" + StringUtils.toString(lts) + ")" +
                ", clustering=" + Arrays.toString(schema.inflateClusteringKey(cd)) +
                ", values=" + Arrays.toString(schema.inflateRegularColumns(vds)) +
                (sds == null ? "" : ", statics=" + Arrays.toString(schema.inflateStaticColumns(sds))) +
diff --git a/harry-core/src/harry/ddl/SchemaGenerators.java b/harry-core/src/harry/ddl/SchemaGenerators.java
index f56c0d5..33cf08a 100644
--- a/harry-core/src/harry/ddl/SchemaGenerators.java
+++ b/harry-core/src/harry/ddl/SchemaGenerators.java
@@ -61,7 +61,8 @@ public class SchemaGenerators
                                                                  ColumnSpec.int16Type,
                                                                  ColumnSpec.int32Type,
                                                                  ColumnSpec.int64Type,
-                                                                 ColumnSpec.booleanType,
+                                                                 // TODO: ByteBufferUtil#objectToBytes doesn't understand booleans
+                                                                 //ColumnSpec.booleanType,
                                                                  ColumnSpec.floatType,
                                                                  ColumnSpec.doubleType,
                                                                  ColumnSpec.asciiType,
@@ -354,7 +355,7 @@ public class SchemaGenerators
     {
         return new SchemaGenerators.Builder(DEFAULT_KEYSPACE_NAME, () -> table)
                .partitionKeySpec(1, 3,
-                                 columnTypes)
+                                 partitionKeyTypes)
                .clusteringKeySpec(1, 3,
                                   clusteringKeyTypes)
                .regularColumnSpec(3, 5,
diff --git a/harry-core/src/harry/dsl/HistoryBuilder.java b/harry-core/src/harry/dsl/HistoryBuilder.java
index fa26277..06e09de 100644
--- a/harry-core/src/harry/dsl/HistoryBuilder.java
+++ b/harry-core/src/harry/dsl/HistoryBuilder.java
@@ -115,6 +115,11 @@ public class HistoryBuilder implements Iterable<ReplayingVisitor.Visit>
         {
             return historyBuilder.position(pd(lts));
         }
+
+        public long maxPosition(long maxLts)
+        {
+            return historyBuilder.partitions;
+        }
     }
 
     private static abstract class Step
diff --git a/harry-core/src/harry/model/AlwaysSamePartitionSelector.java b/harry-core/src/harry/model/AlwaysSamePartitionSelector.java
index 43160fb..3069d9a 100644
--- a/harry-core/src/harry/model/AlwaysSamePartitionSelector.java
+++ b/harry-core/src/harry/model/AlwaysSamePartitionSelector.java
@@ -51,6 +51,11 @@ public class AlwaysSamePartitionSelector extends OpSelectors.PdSelector
         return 0;
     }
 
+    public long maxPosition(long maxLts)
+    {
+        return 0;
+    }
+
     @JsonTypeName("always_same")
     public static class AlwaysSamePartitionSelectorConfiguration implements Configuration.PDSelectorConfiguration
     {
diff --git a/harry-core/src/harry/model/OpSelectors.java b/harry-core/src/harry/model/OpSelectors.java
index 7727828..b9bb34c 100644
--- a/harry-core/src/harry/model/OpSelectors.java
+++ b/harry-core/src/harry/model/OpSelectors.java
@@ -18,11 +18,9 @@
 
 package harry.model;
 
-import java.util.ArrayList;
 import java.util.EnumMap;
 import java.util.List;
 import java.util.Map;
-import java.util.function.LongConsumer;
 
 import harry.core.Configuration;
 import harry.core.VisibleForTesting;
@@ -124,6 +122,7 @@ public interface OpSelectors
 
         // TODO: right now, we can only calculate a position for 64-bit (in other words, full entropy) pds
         public abstract long positionFor(long lts);
+        public abstract long maxPosition(long maxLts);
     }
 
     public static interface PdSelectorFactory
@@ -298,10 +297,31 @@ public interface OpSelectors
 
         public long minLtsFor(long pd)
         {
-            long position = rng.sequenceNumber(pd, PARTITION_DESCRIPTOR_STREAM_ID);
+            long position = positionForPd(pd);
             return minLtsAt(position);
         }
 
+        public long randomVisitedPd(long maxLts, long visitLts, SchemaSpec schema)
+        {
+            long maxPosition = maxPosition(maxLts);
+            if (maxPosition == 0)
+                return schema.adjustPdEntropy(rng.randomNumber(0, PARTITION_DESCRIPTOR_STREAM_ID));
+
+            int idx = RngUtils.asInt(rng.randomNumber(visitLts, maxPosition), 0, (int) (maxPosition - 1));
+            return schema.adjustPdEntropy(rng.randomNumber(idx, PARTITION_DESCRIPTOR_STREAM_ID));
+        }
+
+        public long maxPosition(long maxLts)
+        {
+            long timesCycled = maxLts / switchAfter;
+            long windowStart = timesCycled * windowSize;
+
+            // We have cycled through _current_ window at least once
+            if (maxLts > (windowStart * slideAfterRepeats) + windowSize)
+                return windowStart + windowSize;
+            return windowStart + maxLts % windowSize;
+        }
+
         // TODO: add maxPosition to make it easier/more accessible for the components like sampler, etc
         public long positionFor(long lts)
         {
diff --git a/harry-integration/src/harry/runner/QueryingNoOpChecker.java b/harry-core/src/harry/model/QueryingNoOpValidator.java
similarity index 80%
rename from harry-integration/src/harry/runner/QueryingNoOpChecker.java
rename to harry-core/src/harry/model/QueryingNoOpValidator.java
index 0e7d4fb..a2da871 100644
--- a/harry-integration/src/harry/runner/QueryingNoOpChecker.java
+++ b/harry-core/src/harry/model/QueryingNoOpValidator.java
@@ -16,27 +16,26 @@
  *  limitations under the License.
  */
 
-package harry.runner;
+package harry.model;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import harry.core.Configuration;
 import harry.core.Run;
-import harry.model.Model;
 import harry.model.sut.SystemUnderTest;
 import harry.operations.CompiledStatement;
 import harry.operations.Query;
 
-public class QueryingNoOpChecker implements Model
+/**
+ * A model that can be used to "simply" run random queries. Does not perform validation
+ * of the results that it sees. Useful for increasing concurrency and triggering
+ * exceptions rather than data loss issues.
+ */
+public class QueryingNoOpValidator implements Model
 {
-    public static void init()
-    {
-        Configuration.registerSubtypes(QueryingNoOpCheckerConfig.class);
-    }
-
     private final Run run;
 
-    public QueryingNoOpChecker(Run run)
+    public QueryingNoOpValidator(Run run)
     {
         this.run = run;
     }
@@ -50,7 +49,7 @@ public class QueryingNoOpChecker implements Model
                         compiled.bindings());
     }
 
-    @JsonTypeName("querying_no_op_checker")
+    @JsonTypeName("no_op")
     public static class QueryingNoOpCheckerConfig implements Configuration.ModelConfiguration
     {
         @JsonCreator
@@ -60,7 +59,7 @@ public class QueryingNoOpChecker implements Model
 
         public Model make(Run run)
         {
-            return new QueryingNoOpChecker(run);
+            return new QueryingNoOpValidator(run);
         }
     }
 }
diff --git a/harry-core/src/harry/model/QuiescentChecker.java b/harry-core/src/harry/model/QuiescentChecker.java
index 0246e98..0638915 100644
--- a/harry-core/src/harry/model/QuiescentChecker.java
+++ b/harry-core/src/harry/model/QuiescentChecker.java
@@ -28,9 +28,9 @@ import harry.core.Run;
 import harry.data.ResultSetRow;
 import harry.ddl.SchemaSpec;
 import harry.model.sut.SystemUnderTest;
+import harry.operations.Query;
 import harry.reconciler.Reconciler;
 import harry.runner.DataTracker;
-import harry.operations.Query;
 
 import static harry.generators.DataGenerators.NIL_DESCR;
 
@@ -59,22 +59,17 @@ public class QuiescentChecker implements Model
 
     public void validate(Query query)
     {
+        tracker.beginValidation(query.pd);
         validate(() -> SelectHelper.execute(sut, clock, query), query);
+        tracker.endValidation(query.pd);
     }
 
     protected void validate(Supplier<List<ResultSetRow>> rowsSupplier, Query query)
     {
-        long maxCompeteLts = tracker.maxConsecutiveFinished();
-        long maxSeenLts = tracker.maxStarted();
-
-        assert maxCompeteLts == maxSeenLts : String.format("Runner hasn't settled down yet. " +
-                                                           "Quiescent model can't be reliably used in such cases. " +
-                                                           "Max complete: %d. Max seen: %d",
-                                                           maxCompeteLts, maxSeenLts);
-
         List<ResultSetRow> actualRows = rowsSupplier.get();
         Iterator<ResultSetRow> actual = actualRows.iterator();
-        Reconciler.PartitionState partitionState = reconciler.inflatePartitionState(query.pd, maxSeenLts, query);
+
+        Reconciler.PartitionState partitionState = reconciler.inflatePartitionState(query.pd, tracker, query);
         Collection<Reconciler.RowState> expectedRows = partitionState.rows(query.reverse);
 
         Iterator<Reconciler.RowState> expected = expectedRows.iterator();
@@ -85,7 +80,7 @@ public class QuiescentChecker implements Model
             ResultSetRow actualRowState = actual.next();
             if (actualRowState.cd != partitionState.staticRow().cd)
                 throw new ValidationException(partitionState.toString(schema),
-                                              toString(actualRows, schema),
+                                              toString(actualRows),
                                               "Found a row while model predicts statics only:" +
                                               "\nExpected: %s" +
                                               "\nActual: %s" +
@@ -97,10 +92,9 @@ public class QuiescentChecker implements Model
             {
                 if (actualRowState.vds[i] != NIL_DESCR || actualRowState.lts[i] != NO_TIMESTAMP)
                     throw new ValidationException(partitionState.toString(schema),
-                                                  toString(actualRows, schema),
+                                                  toString(actualRows),
                                                   "Found a row while model predicts statics only:" +
                                                   "\nActual: %s" +
-                                                  "\nQuery: %s" +
                                                   "\nQuery: %s",
                                                   actualRowState, query.toSelectStatement());
             }
@@ -115,7 +109,7 @@ public class QuiescentChecker implements Model
             // TODO: this is not necessarily true. It can also be that ordering is incorrect.
             if (actualRowState.cd != expectedRowState.cd)
                 throw new ValidationException(partitionState.toString(schema),
-                                              toString(actualRows, schema),
+                                              toString(actualRows),
                                               "Found a row in the model that is not present in the resultset:" +
                                               "\nExpected: %s" +
                                               "\nActual: %s" +
@@ -125,7 +119,7 @@ public class QuiescentChecker implements Model
 
             if (!Arrays.equals(actualRowState.vds, expectedRowState.vds))
                 throw new ValidationException(partitionState.toString(schema),
-                                              toString(actualRows, schema),
+                                              toString(actualRows),
                                               "Returned row state doesn't match the one predicted by the model:" +
                                               "\nExpected: %s (%s)" +
                                               "\nActual:   %s (%s)." +
@@ -136,11 +130,12 @@ public class QuiescentChecker implements Model
 
             if (!Arrays.equals(actualRowState.lts, expectedRowState.lts))
                 throw new ValidationException(partitionState.toString(schema),
-                                              toString(actualRows, schema),
+                                              toString(actualRows),
                                               "Timestamps in the row state don't match ones predicted by the model:" +
                                               "\nExpected: %s (%s)" +
                                               "\nActual:   %s (%s)." +
-                                              "\nQuery: %s",
+                                              "\nQuery: %s" +
+                                              "\nMax started: %d, Max finished: %d, %d reordered: %s",
                                               Arrays.toString(expectedRowState.lts), expectedRowState.toString(schema),
                                               Arrays.toString(actualRowState.lts), actualRowState,
                                               query.toSelectStatement());
@@ -152,7 +147,7 @@ public class QuiescentChecker implements Model
         if (actual.hasNext() || expected.hasNext())
         {
             throw new ValidationException(partitionState.toString(schema),
-                                          toString(actualRows, schema),
+                                          toString(actualRows),
                                           "Expected results to have the same number of results, but %s result iterator has more results." +
                                           "\nExpected: %s" +
                                           "\nActual:   %s" +
@@ -173,7 +168,7 @@ public class QuiescentChecker implements Model
     {
         if (!Arrays.equals(staticRow.vds, actualRowState.sds))
             throw new ValidationException(partitionState.toString(schemaSpec),
-                                          toString(actualRows, schemaSpec),
+                                          toString(actualRows),
                                           "Returned static row state doesn't match the one predicted by the model:" +
                                           "\nExpected: %s (%s)" +
                                           "\nActual:   %s (%s)." +
@@ -184,7 +179,7 @@ public class QuiescentChecker implements Model
 
         if (!Arrays.equals(staticRow.lts, actualRowState.slts))
             throw new ValidationException(partitionState.toString(schemaSpec),
-                                          toString(actualRows, schemaSpec),
+                                          toString(actualRows),
                                           "Timestamps in the static row state don't match ones predicted by the model:" +
                                           "\nExpected: %s (%s)" +
                                           "\nActual:   %s (%s)." +
@@ -203,12 +198,12 @@ public class QuiescentChecker implements Model
         return builder.toString();
     }
 
-    public static String toString(List<ResultSetRow> collection, SchemaSpec schema)
+    public static String toString(List<ResultSetRow> collection)
     {
         StringBuilder builder = new StringBuilder();
 
         for (ResultSetRow rowState : collection)
-            builder.append(rowState.toString(schema)).append("\n");
+            builder.append(rowState.toString()).append("\n");
         return builder.toString();
     }
-}
\ No newline at end of file
+}
diff --git a/harry-core/src/harry/model/QuiescentLocalStateCheckerBase.java b/harry-core/src/harry/model/QuiescentLocalStateCheckerBase.java
new file mode 100644
index 0000000..de4ce85
--- /dev/null
+++ b/harry-core/src/harry/model/QuiescentLocalStateCheckerBase.java
@@ -0,0 +1,110 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package harry.model;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NavigableMap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import harry.core.Run;
+import harry.data.ResultSetRow;
+import harry.model.sut.SystemUnderTest;
+import harry.operations.CompiledStatement;
+import harry.operations.Query;
+
+import static harry.model.SelectHelper.resultSetToRow;
+import static harry.model.sut.TokenPlacementModel.Node;
+import static harry.model.sut.TokenPlacementModel.Range;
+import static harry.model.sut.TokenPlacementModel.getReplicas;
+
+public abstract class QuiescentLocalStateCheckerBase extends QuiescentChecker
+{
+    private static final Logger logger = LoggerFactory.getLogger(QuiescentLocalStateCheckerBase.class);
+
+    public final SystemUnderTest sut;
+    public final int rf;
+    private final OpSelectors.PdSelector pdSelector;
+
+    public QuiescentLocalStateCheckerBase(Run run)
+    {
+        this(run, 3);
+    }
+
+    public QuiescentLocalStateCheckerBase(Run run, int rf)
+    {
+        super(run);
+        this.sut = run.sut;
+        this.rf = rf;
+        this.pdSelector = run.pdSelector;
+    }
+
+    @SuppressWarnings("unused")
+    public void validateAll()
+    {
+        NavigableMap<Range, List<Node>> ring = getRing();
+
+        for (int lts = 0; lts < clock.peek(); lts++)
+            validate(Query.selectPartition(schema, pdSelector.pd(lts, schema), false), ring);
+    }
+
+    @Override
+    public void validate(Query query)
+    {
+        NavigableMap<Range, List<Node>> ring = getRing();
+        tracker.beginValidation(query.pd);
+        validate(query, ring);
+        tracker.endValidation(query.pd);
+    }
+
+    protected void validate(Query query, NavigableMap<Range, List<Node>> ring)
+    {
+        CompiledStatement compiled = query.toSelectStatement();
+        List<Node> replicas = getReplicas(ring, token(query.pd));
+
+        logger.trace("Predicted {} as replicas for {}. Ring: {}", replicas, query.pd, ring);
+        for (Node node : replicas)
+        {
+            try
+            {
+                validate(() -> {
+                    Object[][] objects = executeNodeLocal(compiled.cql(), node, compiled.bindings());
+
+                    List<ResultSetRow> result = new ArrayList<>();
+                    for (Object[] obj : objects)
+                        result.add(resultSetToRow(query.schemaSpec, clock, obj));
+
+                    return result;
+                }, query);
+            }
+            catch (ValidationException e)
+            {
+                throw new AssertionError(String.format("Caught error while validating replica %s of replica set %s",
+                                                       node, replicas),
+                                         e);
+            }
+        }
+    }
+
+    protected abstract NavigableMap<Range, List<Node>> getRing();
+    protected abstract long token(long pd);
+    protected abstract Object[][] executeNodeLocal(String statement, Node node, Object... bindings);
+}
diff --git a/harry-integration/src/harry/runner/RepairingLocalStateValidator.java b/harry-core/src/harry/model/RepairingLocalStateValidator.java
similarity index 50%
rename from harry-integration/src/harry/runner/RepairingLocalStateValidator.java
rename to harry-core/src/harry/model/RepairingLocalStateValidator.java
index 71c1535..ff93b29 100644
--- a/harry-integration/src/harry/runner/RepairingLocalStateValidator.java
+++ b/harry-core/src/harry/model/RepairingLocalStateValidator.java
@@ -16,42 +16,43 @@
  *  limitations under the License.
  */
 
-package harry.runner;
+package harry.model;
 
-import java.util.ArrayList;
-import java.util.List;
+import java.util.function.LongPredicate;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import harry.core.Configuration;
 import harry.core.Run;
-import harry.data.ResultSetRow;
-import harry.model.Model;
-import harry.model.OpSelectors;
-import harry.model.QuiescentChecker;
-import harry.model.sut.InJvmSut;
-import harry.model.sut.SystemUnderTest;
-import harry.operations.CompiledStatement;
-import harry.operations.Query;
+import harry.model.sut.injvm.InJvmSut;
 import harry.visitors.AllPartitionsValidator;
 import harry.visitors.Visitor;
 
-import static harry.model.SelectHelper.resultSetToRow;
-
+/**
+ * Validator similar to {@link AllPartitionsValidator}, but performs
+ * repair before checking node states.
+ *
+ * Can be useful for testing repair, bootstrap, and streaming code.
+ */
 public class RepairingLocalStateValidator extends AllPartitionsValidator
 {
-    public static void init()
+    private final InJvmSut inJvmSut;
+    private final OpSelectors.MonotonicClock clock;
+
+    public static Configuration.VisitorConfiguration factory(int concurrency, LongPredicate condition, Model.ModelFactory modelFactory)
     {
-        Configuration.registerSubtypes(RepairingLocalStateValidatorConfiguration.class,
-                                       QuiescentCheckerConfig.class);
+        return (r) -> new RepairingLocalStateValidator(r, concurrency, condition, modelFactory);
     }
 
-    private final InJvmSut inJvmSut;
-    private final OpSelectors.MonotonicClock clock;
-    public RepairingLocalStateValidator(int concurrency, int triggerAfter, Run run, Model.ModelFactory modelFactory)
+    public RepairingLocalStateValidator(Run run, int concurrency, long triggerAfter, Model.ModelFactory modelFactory)
     {
-        super(concurrency, triggerAfter, run, modelFactory);
+        this(run, concurrency, (lts) -> lts > 0 && lts % triggerAfter == 0, modelFactory);
+    }
+
+    public RepairingLocalStateValidator(Run run, int concurrency, LongPredicate condition, Model.ModelFactory modelFactory)
+    {
+        super(run, concurrency, condition, modelFactory);
         this.inJvmSut = (InJvmSut) run.sut;
         this.clock = run.clock;
     }
@@ -59,12 +60,10 @@ public class RepairingLocalStateValidator extends AllPartitionsValidator
     public void visit()
     {
         long lts = clock.peek();
-        if (lts > 0 && lts % triggerAfter == 0)
+        if (condition.test(lts))
         {
             System.out.println("Starting repair...");
-
             inJvmSut.cluster().stream().forEach((instance) -> instance.nodetool("repair", "--full"));
-
             System.out.println("Validating partitions...");
             super.visit();
         }
@@ -89,55 +88,7 @@ public class RepairingLocalStateValidator extends AllPartitionsValidator
 
         public Visitor make(Run run)
         {
-            return new RepairingLocalStateValidator(concurrency, trigger_after, run, modelConfiguration);
-        }
-    }
-
-    public static class QuiescentLocalStateChecker extends QuiescentChecker
-    {
-        public final InJvmSut inJvmSut;
-
-        public QuiescentLocalStateChecker(Run run)
-        {
-            super(run);
-            assert run.sut instanceof InJvmSut;
-
-            this.inJvmSut = (InJvmSut) run.sut;
-        }
-
-        @Override
-        public void validate(Query query)
-        {
-            CompiledStatement compiled = query.toSelectStatement();
-            int[] replicas = inJvmSut.getReplicasFor(schema.inflatePartitionKey(query.pd), schema.keyspace, schema.table);
-            for (int node : replicas)
-            {
-                validate(() -> {
-                    Object[][] objects = inJvmSut.execute(compiled.cql(),
-                                                          SystemUnderTest.ConsistencyLevel.NODE_LOCAL,
-                                                          node,
-                                                          compiled.bindings());
-                    List<ResultSetRow> result = new ArrayList<>();
-                    for (Object[] obj : objects)
-                        result.add(resultSetToRow(query.schemaSpec, clock, obj));
-
-                    return result;
-                }, query);
-            }
-        }
-    }
-
-    @JsonTypeName("quiescent_local_state_checker")
-    public static class QuiescentCheckerConfig implements Configuration.ModelConfiguration
-    {
-        @JsonCreator
-        public QuiescentCheckerConfig()
-        {
-        }
-
-        public Model make(Run run)
-        {
-            return new QuiescentLocalStateChecker(run);
+            return new RepairingLocalStateValidator(run, concurrency, trigger_after, modelConfiguration);
         }
     }
 }
diff --git a/harry-core/src/harry/model/clock/OffsetClock.java b/harry-core/src/harry/model/clock/OffsetClock.java
index 6e8f891..2ea3576 100644
--- a/harry-core/src/harry/model/clock/OffsetClock.java
+++ b/harry-core/src/harry/model/clock/OffsetClock.java
@@ -28,12 +28,18 @@ import harry.model.OpSelectors;
 
 public class OffsetClock implements OpSelectors.MonotonicClock
 {
-    final AtomicLong lts = new AtomicLong(ApproximateMonotonicClock.START_VALUE);
+    final AtomicLong lts;
 
     private final long base;
 
     public OffsetClock(long base)
     {
+        this(ApproximateMonotonicClock.START_VALUE, base);
+    }
+
+    public OffsetClock(long startValue, long base)
+    {
+        this.lts = new AtomicLong(startValue);
         this.base = base;
     }
 
@@ -59,23 +65,26 @@ public class OffsetClock implements OpSelectors.MonotonicClock
 
     public Configuration.ClockConfiguration toConfig()
     {
-        throw new RuntimeException("not implemented");
+        return new OffsetClockConfiguration(lts.get(), base);
     }
 
     @JsonTypeName("offset")
     public static class OffsetClockConfiguration implements Configuration.ClockConfiguration
     {
         public final long offset;
+        public final long base;
 
         @JsonCreator
-        public OffsetClockConfiguration(@JsonProperty("offset") int offset)
+        public OffsetClockConfiguration(@JsonProperty("offset") long offset,
+                                        @JsonProperty(value = "base", defaultValue = "0") long base)
         {
             this.offset = offset;
+            this.base = base;
         }
 
         public OpSelectors.MonotonicClock make()
         {
-            return new OffsetClock(offset);
+            return new OffsetClock(base, offset);
         }
     }
 }
diff --git a/harry-core/src/harry/model/sut/TokenPlacementModel.java b/harry-core/src/harry/model/sut/TokenPlacementModel.java
new file mode 100644
index 0000000..fe161f9
--- /dev/null
+++ b/harry-core/src/harry/model/sut/TokenPlacementModel.java
@@ -0,0 +1,218 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package harry.model.sut;
+
+import java.net.InetAddress;
+import java.util.*;
+
+public class TokenPlacementModel
+{
+    /**
+     * Replicate ranges to rf nodes.
+     */
+    public static NavigableMap<Range, List<Node>> replicate(List<Node> nodes, int rf)
+    {
+        nodes.sort(Node::compareTo);
+        List<Range> ranges = toRanges(nodes);
+        return replicate(ranges, nodes, rf);
+    }
+
+    public static List<Node> getReplicas(NavigableMap<TokenPlacementModel.Range, List<TokenPlacementModel.Node>> ring, long token)
+    {
+        return ring.get(ring.floorKey(new Range(token, token, true)));
+    }
+
+    public static NavigableMap<Range, List<Node>> replicate(List<Range> ranges, List<Node> nodes, int rf)
+    {
+        NavigableMap<Range, List<Node>> replication = new TreeMap<>();
+        for (Range range : ranges)
+        {
+            Set<String> names = new HashSet<>();
+            List<Node> replicas = new ArrayList<>();
+            int idx = primaryReplica(nodes, range);
+            if (idx >= 0)
+            {
+                for (int i = idx; i < nodes.size() && replicas.size() < rf; i++)
+                    addIfUnique(replicas, names, nodes.get(i));
+
+                for (int i = 0; replicas.size() < rf && i < idx; i++)
+                    addIfUnique(replicas, names, nodes.get(i));
+                if (range.start == Long.MIN_VALUE)
+                    replication.put(ranges.get(ranges.size() - 1), replicas);
+                replication.put(range, replicas);
+            }
+            else
+            {
+                // if the range end is larger than the highest assigned token, then treat it
+                // as part of the wraparound and replicate it to the same nodes as the first
+                // range. This is most likely caused by a decommission removing the node with
+                // the largest token.
+                replication.put(range, replication.get(ranges.get(0)));
+            }
+        }
+        return replication;
+    }
+
+    public static void addIfUnique(List<Node> nodes, Set<String> names, Node node)
+    {
+        if (names.contains(node.id))
+            return;
+        nodes.add(node);
+        names.add(node.id);
+    }
+
+    /**
+     * Finds a primary replica
+     */
+    public static int primaryReplica(List<Node> nodes, Range range)
+    {
+        for (int i = 0; i < nodes.size(); i++)
+        {
+            if (range.end != Long.MIN_VALUE && nodes.get(i).token >= range.end)
+                return i;
+        }
+        return -1;
+    }
+
+    /**
+     * Generates token ranges from the list of nodes
+     */
+    public static List<Range> toRanges(List<Node> nodes)
+    {
+        List<Long> tokens = new ArrayList<>();
+        for (Node node : nodes)
+        {
+            if (node.token != Long.MIN_VALUE)
+                tokens.add(node.token);
+        }
+        tokens.add(Long.MIN_VALUE);
+        tokens.sort(Long::compareTo);
+
+        List<Range> ranges = new ArrayList<>(tokens.size() + 1);
+        long prev = tokens.get(0);
+        for (int i = 1; i < tokens.size(); i++)
+        {
+            long current = tokens.get(i);
+            ranges.add(new Range(prev, current));
+            prev = current;
+        }
+        ranges.add(new Range(prev, Long.MIN_VALUE));
+        return Collections.unmodifiableList(ranges);
+    }
+
+    public static List<Node> peerStateToNodes(Object[][] resultset)
+    {
+        List<Node> nodes = new ArrayList<>();
+        for (Object[] row : resultset)
+        {
+            InetAddress address = (InetAddress) row[0];
+            Set<String> tokens = (Set<String>) row[1];
+            for (String token : tokens)
+                nodes.add(new Node(Long.parseLong(token), address.toString()));
+        }
+        return nodes;
+    }
+
+    public static class Range implements Comparable<Range>
+    {
+        public final long start;
+        public final long end;
+
+        public Range(long start, long end)
+        {
+            this(start, end, false);
+        }
+
+        public Range(long start, long end, boolean skipAssert)
+        {
+            assert skipAssert || (end > start || end == Long.MIN_VALUE);
+            this.start = start;
+            this.end = end;
+        }
+
+        public boolean contains(long token)
+        {
+            return token > start && (token <= end || end == Long.MIN_VALUE);
+        }
+
+        public int compareTo(Range o)
+        {
+            int res = Long.compare(start, o.start);
+            if (res == 0)
+                return Long.compare(end, o.end);
+            return res;
+        }
+
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Range range = (Range) o;
+            return start == range.start && end == range.end;
+        }
+
+        public int hashCode()
+        {
+            return Objects.hash(start, end);
+        }
+
+        public String toString()
+        {
+            return "(" +
+                   "" + (start == Long.MIN_VALUE ? "MIN" : start) +
+                   ", " + (end == Long.MAX_VALUE ? "MAX" : end) +
+                   ']';
+        }
+    }
+
+    public static class Node implements Comparable<Node>
+    {
+        public final long token;
+        public final String id;
+
+        public Node(long token, String id)
+        {
+            this.token = token;
+            this.id = id;
+        }
+
+        public boolean equals(Object o)
+        {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Node node = (Node) o;
+            return Objects.equals(id, node.id);
+        }
+
+        public int hashCode()
+        {
+            return Objects.hash(id);
+        }
+
+        public int compareTo(Node o)
+        {
+            return Long.compare(token, o.token);
+        }
+
+        public String toString()
+        {
+            return "" + id + "@" + token;
+        }
+    }
+}
diff --git a/harry-integration/src/harry/model/sut/InJVMTokenAwareVisitExecutor.java b/harry-core/src/harry/model/sut/injvm/InJVMTokenAwareVisitExecutor.java
similarity index 53%
rename from harry-integration/src/harry/model/sut/InJVMTokenAwareVisitExecutor.java
rename to harry-core/src/harry/model/sut/injvm/InJVMTokenAwareVisitExecutor.java
index 6bc0705..17a91d7 100644
--- a/harry-integration/src/harry/model/sut/InJVMTokenAwareVisitExecutor.java
+++ b/harry-core/src/harry/model/sut/injvm/InJVMTokenAwareVisitExecutor.java
@@ -16,43 +16,61 @@
  *  limitations under the License.
  */
 
-package harry.model.sut;
+package harry.model.sut.injvm;
 
-import java.util.Random;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NavigableMap;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import harry.core.Run;
 import harry.ddl.SchemaSpec;
+import harry.model.sut.SystemUnderTest;
+import harry.model.sut.TokenPlacementModel;
 import harry.operations.CompiledStatement;
+import harry.util.ByteUtils;
+import harry.util.TokenUtil;
 import harry.visitors.GeneratingVisitor;
 import harry.visitors.LoggingVisitor;
 import harry.visitors.OperationExecutor;
+import harry.visitors.VisitExecutor;
 import harry.visitors.Visitor;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.IInstance;
+
+import static harry.model.sut.TokenPlacementModel.peerStateToNodes;
+import static harry.model.sut.TokenPlacementModel.getReplicas;
 
 public class InJVMTokenAwareVisitExecutor extends LoggingVisitor.LoggingVisitorExecutor
 {
-    public static void init()
-    {
-        harry.core.Configuration.registerSubtypes(Configuration.class);
-    }
-
     private final InJvmSut sut;
+    private final int rf;
     private final SystemUnderTest.ConsistencyLevel cl;
     private final SchemaSpec schema;
-    private final int maxRetries = 10;
+    private final int MAX_RETRIES = 10;
+
+    public static Function<Run, VisitExecutor> factory(OperationExecutor.RowVisitorFactory rowVisitorFactory,
+                                                       SystemUnderTest.ConsistencyLevel cl,
+                                                       int rf)
+    {
+        return (run) -> new InJVMTokenAwareVisitExecutor(run, rowVisitorFactory, cl, rf);
+    }
 
     public InJVMTokenAwareVisitExecutor(Run run,
                                         OperationExecutor.RowVisitorFactory rowVisitorFactory,
-                                        SystemUnderTest.ConsistencyLevel cl)
+                                        SystemUnderTest.ConsistencyLevel cl,
+                                        int rf)
     {
         super(run, rowVisitorFactory.make(run));
         this.sut = (InJvmSut) run.sut;
         this.schema = run.schemaSpec;
         this.cl = cl;
+        this.rf = rf;
     }
 
     @Override
@@ -66,20 +84,26 @@ public class InJVMTokenAwareVisitExecutor extends LoggingVisitor.LoggingVisitorE
         if (sut.isShutdown())
             throw new IllegalStateException("System under test is shut down");
 
-        if (retries > this.maxRetries)
+        if (retries > this.MAX_RETRIES)
             throw new IllegalStateException(String.format("Can not execute statement %s after %d retries", statement, retries));
 
-        Object[] partitionKey =  schema.inflatePartitionKey(pd);
-        int[] replicas = sut.getReplicasFor(partitionKey, schema.keyspace, schema.table);
-        // TODO: find a better source of entropy
-        int replica = replicas[new Random(lts).nextInt(replicas.length)];
+        Object[] pk =  schema.inflatePartitionKey(pd);
+        List<TokenPlacementModel.Node> replicas = getReplicas(getRing(), TokenUtil.token(ByteUtils.compose(ByteUtils.objectsToBytes(pk))));
+
+        TokenPlacementModel.Node replica = replicas.get((int) (lts % replicas.size()));
         if (cl == SystemUnderTest.ConsistencyLevel.NODE_LOCAL)
         {
-            future.complete(sut.cluster.get(replica).executeInternal(statement.cql(), statement.bindings()));
+            future.complete(executeNodeLocal(statement.cql(), replica, statement.bindings()));
         }
         else
         {
-            CompletableFuture.supplyAsync(() -> sut.cluster.coordinator(replica).execute(statement.cql(), InJvmSut.toApiCl(cl), statement.bindings()), executor)
+            CompletableFuture.supplyAsync(() ->  sut.cluster
+                                                 .stream()
+                                                 .filter((n) -> n.config().broadcastAddress().toString().contains(replica.id))
+                                                 .findFirst()
+                                                 .get()
+                                                 .coordinator()
+                                                 .execute(statement.cql(), InJvmSut.toApiCl(cl), statement.bindings()), executor)
                              .whenComplete((res, t) ->
                                            {
                                                if (t != null)
@@ -90,24 +114,47 @@ public class InJVMTokenAwareVisitExecutor extends LoggingVisitor.LoggingVisitorE
         }
     }
 
+    protected NavigableMap<TokenPlacementModel.Range, List<TokenPlacementModel.Node>> getRing()
+    {
+        List<TokenPlacementModel.Node> other = peerStateToNodes(sut.cluster.coordinator(1).execute("select peer, tokens from system.peers", ConsistencyLevel.ONE));
+        List<TokenPlacementModel.Node> self = peerStateToNodes(sut.cluster.coordinator(1).execute("select broadcast_address, tokens from system.local", ConsistencyLevel.ONE));
+        List<TokenPlacementModel.Node> all = new ArrayList<>();
+        all.addAll(self);
+        all.addAll(other);
+        return TokenPlacementModel.replicate(all, rf);
+    }
+
+    protected Object[][] executeNodeLocal(String statement, TokenPlacementModel.Node node, Object... bindings)
+    {
+        IInstance instance = sut.cluster
+                             .stream()
+                             .filter((n) -> n.config().broadcastAddress().toString().contains(node.id))
+                             .findFirst()
+                             .get();
+        return instance.executeInternal(statement, bindings);
+    }
+
+
     @JsonTypeName("in_jvm_token_aware")
     public static class Configuration implements harry.core.Configuration.VisitorConfiguration
     {
         public final harry.core.Configuration.RowVisitorConfiguration row_visitor;
         public final SystemUnderTest.ConsistencyLevel consistency_level;
-
+        public final int rf;
         @JsonCreator
         public Configuration(@JsonProperty("row_visitor") harry.core.Configuration.RowVisitorConfiguration rowVisitor,
-                             @JsonProperty("consistency_level") SystemUnderTest.ConsistencyLevel consistencyLevel)
+                             @JsonProperty("consistency_level") SystemUnderTest.ConsistencyLevel consistencyLevel,
+                             @JsonProperty("rf") int rf)
         {
             this.row_visitor = rowVisitor;
             this.consistency_level = consistencyLevel;
+            this.rf = rf;
         }
 
         @Override
         public Visitor make(Run run)
         {
-            return new GeneratingVisitor(run, new InJVMTokenAwareVisitExecutor(run, row_visitor, consistency_level));
+            return new GeneratingVisitor(run, new InJVMTokenAwareVisitExecutor(run, row_visitor, consistency_level, rf));
         }
     }
 }
\ No newline at end of file
diff --git a/harry-integration/src/harry/runner/HarryRunnerJvm.java b/harry-core/src/harry/model/sut/injvm/InJvmSut.java
similarity index 61%
rename from harry-integration/src/harry/runner/HarryRunnerJvm.java
rename to harry-core/src/harry/model/sut/injvm/InJvmSut.java
index ed4ea46..46eaf9a 100644
--- a/harry-integration/src/harry/runner/HarryRunnerJvm.java
+++ b/harry-core/src/harry/model/sut/injvm/InJvmSut.java
@@ -16,28 +16,20 @@
  *  limitations under the License.
  */
 
-package harry.runner;
+package harry.model.sut.injvm;
 
-import harry.core.Configuration;
-import harry.model.sut.InJvmSut;
+import org.apache.cassandra.distributed.api.ICluster;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
 
-import java.io.File;
-
-public class HarryRunnerJvm extends HarryRunner {
-
-    public static void main(String[] args) throws Throwable {
-        InJvmSut.init();
-
-        HarryRunnerJvm runner = new HarryRunnerJvm();
-        File configFile = runner.loadConfig(args);
-
-        Configuration configuration = Configuration.fromFile(configFile);
-        runner.run(configuration);
+public class InJvmSut extends InJvmSutBase<IInvokableInstance, ICluster<IInvokableInstance>>
+{
+    public InJvmSut(ICluster<IInvokableInstance> cluster)
+    {
+        super(cluster, 10);
     }
 
-
-    @Override
-    public void beforeRun(Runner.TimedRunner runner) {
-
+    public InJvmSut(ICluster<IInvokableInstance> cluster, int threads)
+    {
+        super(cluster, threads);
     }
 }
diff --git a/harry-integration/src/harry/model/sut/InJvmSutBase.java b/harry-core/src/harry/model/sut/injvm/InJvmSutBase.java
similarity index 80%
rename from harry-integration/src/harry/model/sut/InJvmSutBase.java
rename to harry-core/src/harry/model/sut/injvm/InJvmSutBase.java
index aa2608f..500a472 100644
--- a/harry-integration/src/harry/model/sut/InJvmSutBase.java
+++ b/harry-core/src/harry/model/sut/injvm/InJvmSutBase.java
@@ -16,12 +16,14 @@
  *  limitations under the License.
  */
 
-package harry.model.sut;
+package harry.model.sut.injvm;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
-import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -36,21 +38,15 @@ import org.slf4j.LoggerFactory;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import harry.core.Configuration;
-import org.apache.cassandra.distributed.api.Feature;
+import harry.model.sut.SystemUnderTest;
 import org.apache.cassandra.distributed.api.ICluster;
 import org.apache.cassandra.distributed.api.IInstance;
 import org.apache.cassandra.distributed.api.IInstanceConfig;
 import org.apache.cassandra.distributed.api.IMessage;
 import org.apache.cassandra.distributed.api.IMessageFilters;
-import relocated.shaded.com.google.common.collect.Iterators;
 
 public class InJvmSutBase<NODE extends IInstance, CLUSTER extends ICluster<NODE>> implements SystemUnderTest.FaultInjectingSut
 {
-    public static void init()
-    {
-        Configuration.registerSubtypes(InJvmSutBaseConfiguration.class);
-    }
-
     private static final Logger logger = LoggerFactory.getLogger(InJvmSutBase.class);
 
     // TODO: shut down properly
@@ -120,11 +116,11 @@ public class InJvmSutBase<NODE extends IInstance, CLUSTER extends ICluster<NODE>
             }
             else if (statement.contains("SELECT"))
             {
-                return Iterators.toArray(cluster
-                                         // round-robin
-                                         .coordinator(coordinator)
-                                         .executeWithPaging(statement, toApiCl(cl), 1, bindings),
-                                         Object[].class);
+                return toArray(cluster
+                               // round-robin
+                               .coordinator(coordinator)
+                               .executeWithPaging(statement, toApiCl(cl), 1, bindings),
+                               Object[].class);
             }
             else
             {
@@ -139,10 +135,6 @@ public class InJvmSutBase<NODE extends IInstance, CLUSTER extends ICluster<NODE>
             // TODO: find a better way to work around timeouts
             if (t.getMessage().contains("timed out"))
                 return execute(statement, cl, coordinator, bindings);
-
-            logger.error(String.format("Caught error while trying execute statement %s (%s): %s",
-                                       statement, Arrays.toString(bindings), t.getMessage()),
-                         t);
             throw t;
         }
     }
@@ -242,23 +234,7 @@ public class InJvmSutBase<NODE extends IInstance, CLUSTER extends ICluster<NODE>
 
             CLUSTER cluster;
 
-            cluster = cluster((cfg) -> {
-                                  // TODO: make this configurable
-                                  cfg.with(Feature.NETWORK, Feature.GOSSIP, Feature.NATIVE_PROTOCOL)
-                                     .set("row_cache_size_in_mb", 10L)
-                                     .set("index_summary_capacity_in_mb", 10L)
-                                     .set("counter_cache_size_in_mb", 10L)
-                                     .set("key_cache_size_in_mb", 10L)
-                                     .set("file_cache_size_in_mb", 10)
-                                     .set("memtable_heap_space_in_mb", 128)
-                                     .set("memtable_offheap_space_in_mb", 128)
-                                     .set("memtable_flush_writers", 1)
-                                     .set("concurrent_compactors", 1)
-                                     .set("concurrent_reads", 5)
-                                     .set("concurrent_writes", 5)
-                                     .set("compaction_throughput_mb_per_sec", 10)
-                                     .set("hinted_handoff_enabled", false);
-                              },
+            cluster = cluster(defaultConfig(),
                               nodes,
                               new File(root));
 
@@ -267,6 +243,25 @@ public class InJvmSutBase<NODE extends IInstance, CLUSTER extends ICluster<NODE>
         }
     }
 
+    public static Consumer<IInstanceConfig> defaultConfig()
+    {
+        return (cfg) -> {
+            // TODO: make this configurable
+            cfg.set("row_cache_size", "256MiB")
+               .set("index_summary_capacity", "256MiB")
+               .set("counter_cache_size", "256MiB")
+               .set("key_cache_size", "256MiB")
+               .set("file_cache_size", "256MiB")
+               .set("memtable_heap_space", "512MiB")
+               .set("memtable_offheap_space", "512MiB")
+               .set("memtable_flush_writers", 2)
+               .set("concurrent_compactors", 2)
+               .set("concurrent_reads", 5)
+               .set("concurrent_writes", 5)
+               .set("compaction_throughput_mb_per_sec", 10)
+               .set("hinted_handoff_enabled", false);
+        };
+    }
     public static org.apache.cassandra.distributed.api.ConsistencyLevel toApiCl(ConsistencyLevel cl)
     {
         switch (cl)
@@ -278,4 +273,13 @@ public class InJvmSutBase<NODE extends IInstance, CLUSTER extends ICluster<NODE>
         }
         throw new IllegalArgumentException("Don't know a CL: " + cl);
     }
+
+    public static <T> T[] toArray(Iterator<? extends T> iterator, Class<T> klass) {
+        List<T> list = new ArrayList<>();
+        while (iterator.hasNext())
+            list.add(iterator.next());
+        T[] arr = (T[]) java.lang.reflect.Array.newInstance(klass, list.size());
+        return list.toArray(arr);
+    }
+
 }
diff --git a/harry-core/src/harry/model/sut/injvm/MixedVersionInJvmSut.java b/harry-core/src/harry/model/sut/injvm/MixedVersionInJvmSut.java
new file mode 100644
index 0000000..33bf602
--- /dev/null
+++ b/harry-core/src/harry/model/sut/injvm/MixedVersionInJvmSut.java
@@ -0,0 +1,74 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package harry.model.sut.injvm;
+
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.distributed.api.ICluster;
+import org.apache.cassandra.distributed.api.IUpgradeableInstance;
+import org.apache.cassandra.distributed.shared.Versions;
+
+/**
+ * Make sure to set -Dcassandra.test.dtest_jar_path when using this class
+ */
+public class MixedVersionInJvmSut extends InJvmSutBase<IUpgradeableInstance, ICluster<IUpgradeableInstance>>
+{
+    private static final Logger logger = LoggerFactory.getLogger(MixedVersionInJvmSut.class);
+    private final Versions.Version initialVersion;
+    private final List<Versions.Version> versions;
+
+    public MixedVersionInJvmSut(ICluster<IUpgradeableInstance> cluster, Versions.Version initialVersion, List<Versions.Version> versions)
+    {
+        super(cluster, 10);
+        this.initialVersion = initialVersion;
+        this.versions = versions;
+    }
+
+
+    @Override
+    public void afterSchemaInit()
+    {
+        for (int i = 1; i <= cluster.size(); i++)
+        {
+            Versions.Version v = versions.get(i - 1);
+            if (!v.equals(initialVersion))
+            {
+                logger.info("Upgrading {} node from {} to {}", i, initialVersion, v);
+                IUpgradeableInstance instance = cluster.get(i);
+                try
+                {
+                    instance.shutdown().get();
+                }
+                catch (Throwable e)
+                {
+                    throw new RuntimeException(e);
+                }
+                instance.setVersion(v);
+                instance.startup();
+            }
+            else
+            {
+                logger.info("Skipping {} node upgrade, since it is already at the required version ({})", i, initialVersion);
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/harry-core/src/harry/model/sut/injvm/QuiescentLocalStateChecker.java b/harry-core/src/harry/model/sut/injvm/QuiescentLocalStateChecker.java
new file mode 100644
index 0000000..0deb2c8
--- /dev/null
+++ b/harry-core/src/harry/model/sut/injvm/QuiescentLocalStateChecker.java
@@ -0,0 +1,76 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package harry.model.sut.injvm;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NavigableMap;
+
+import harry.core.Run;
+import harry.model.QuiescentLocalStateCheckerBase;
+import harry.model.sut.TokenPlacementModel;
+import harry.util.ByteUtils;
+import harry.util.TokenUtil;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.IInstance;
+
+public class QuiescentLocalStateChecker extends QuiescentLocalStateCheckerBase
+{
+    public QuiescentLocalStateChecker(Run run)
+    {
+        super(run);
+    }
+
+    public static ModelFactory factory(int rf)
+    {
+        return (run) -> new QuiescentLocalStateChecker(run, rf);
+    }
+
+    public QuiescentLocalStateChecker(Run run, int rf)
+    {
+        super(run, rf);
+    }
+
+    @Override
+    protected NavigableMap<TokenPlacementModel.Range, List<TokenPlacementModel.Node>> getRing()
+    {
+        List<TokenPlacementModel.Node> other = TokenPlacementModel.peerStateToNodes(((InJvmSutBase<?, ?>) sut).cluster.coordinator(1).execute("select peer, tokens from system.peers", ConsistencyLevel.ONE));
+        List<TokenPlacementModel.Node> self = TokenPlacementModel.peerStateToNodes(((InJvmSutBase<?, ?>) sut).cluster.coordinator(1).execute("select broadcast_address, tokens from system.local", ConsistencyLevel.ONE));
+        List<TokenPlacementModel.Node> all = new ArrayList<>();
+        all.addAll(self);
+        all.addAll(other);
+        return TokenPlacementModel.replicate(all, rf);
+    }
+
+    @Override
+    protected Object[][] executeNodeLocal(String statement, TokenPlacementModel.Node node, Object... bindings)
+    {
+        IInstance instance = ((InJvmSutBase<?, ?>) sut).cluster
+                             .stream()
+                             .filter((n) -> n.config().broadcastAddress().toString().contains(node.id))
+                             .findFirst()
+                             .get();
+        return instance.executeInternal(statement, bindings);
+    }
+
+    protected long token(long pd)
+    {
+        return TokenUtil.token(ByteUtils.compose(ByteUtils.objectsToBytes(schema.inflatePartitionKey(pd))));
+    }
+}
diff --git a/harry-core/src/harry/reconciler/Reconciler.java b/harry-core/src/harry/reconciler/Reconciler.java
index 45649be..e4970dd 100644
--- a/harry-core/src/harry/reconciler/Reconciler.java
+++ b/harry-core/src/harry/reconciler/Reconciler.java
@@ -37,8 +37,10 @@ import harry.ddl.SchemaSpec;
 import harry.model.OpSelectors;
 import harry.operations.Query;
 import harry.operations.QueryGenerator;
+import harry.runner.DataTracker;
 import harry.util.BitSet;
 import harry.util.Ranges;
+import harry.util.StringUtils;
 import harry.visitors.GeneratingVisitor;
 import harry.visitors.LtsVisitor;
 import harry.visitors.ReplayingVisitor;
@@ -75,8 +77,7 @@ public class Reconciler
              (processor) -> new GeneratingVisitor(run, processor));
     }
 
-    public Reconciler(Run run,
-                      Function<VisitExecutor, LtsVisitor> ltsVisitorFactory)
+    public Reconciler(Run run, Function<VisitExecutor, LtsVisitor> ltsVisitorFactory)
     {
         this.descriptorSelector = run.descriptorSelector;
         this.pdSelector = run.pdSelector;
@@ -85,11 +86,11 @@ public class Reconciler
         this.visitorFactory = ltsVisitorFactory;
     }
 
-    private final long debugCd = Long.getLong("harry.reconciler.debug_cd", -1L);
+    private final long debugCd = Long.getLong("debug_cd", -1L);
 
-    public PartitionState inflatePartitionState(final long pd, long maxLts, Query query)
+    public PartitionState inflatePartitionState(final long pd, DataTracker tracker, Query query)
     {
-        PartitionState partitionState = new PartitionState();
+        PartitionState partitionState = new PartitionState(pd);
 
         class Processor extends VisitExecutor
         {
@@ -260,10 +261,12 @@ public class Reconciler
         LtsVisitor visitor = visitorFactory.apply(new Processor());
 
         long currentLts = pdSelector.minLtsFor(pd);
-
-        while (currentLts <= maxLts && currentLts >= 0)
+        long maxStarted = tracker.maxStarted();
+        while (currentLts <= maxStarted && currentLts >= 0)
         {
-            visitor.visit(currentLts);
+            if (tracker.isFinished(currentLts))
+                visitor.visit(currentLts);
+
             currentLts = pdSelector.nextLts(currentLts);
         }
 
@@ -272,15 +275,18 @@ public class Reconciler
 
     public class PartitionState implements Iterable<RowState>
     {
+        private final long pd;
         private final NavigableMap<Long, RowState> rows;
         private RowState staticRow;
 
-        private PartitionState()
+        private PartitionState(long pd)
         {
+            this.pd = pd;
             rows = new TreeMap<>();
             if (!schema.staticColumns.isEmpty())
             {
-                staticRow = new RowState(STATIC_CLUSTERING,
+                staticRow = new RowState(this,
+                                         STATIC_CLUSTERING,
                                          arr(schema.staticColumns.size(), NIL_DESCR),
                                          arr(schema.staticColumns.size(), NO_TIMESTAMP));
             }
@@ -308,9 +314,9 @@ public class Reconciler
                 return;
 
             Iterator<Map.Entry<Long, RowState>> iter = rows.subMap(range.minBound, range.minInclusive,
-                                                                   range.maxBound, range.maxInclusive)
-                                                           .entrySet()
-                                                           .iterator();
+                                                                              range.maxBound, range.maxInclusive)
+                                                                      .entrySet()
+                                                                      .iterator();
             while (iter.hasNext())
             {
                 Map.Entry<Long, RowState> e = iter.next();
@@ -357,7 +363,7 @@ public class Reconciler
                     }
                 }
 
-                currentState = new RowState(cd, vdsCopy, ltss);
+                currentState = new RowState(this, cd, vdsCopy, ltss);
             }
             else
             {
@@ -489,14 +495,17 @@ public class Reconciler
     public static class RowState
     {
         public boolean hasPrimaryKeyLivenessInfo = false;
+        public final PartitionState partitionState;
         public final long cd;
         public final long[] vds;
         public final long[] lts;
 
-        public RowState(long cd,
+        public RowState(PartitionState partitionState,
+                        long cd,
                         long[] vds,
                         long[] lts)
         {
+            this.partitionState = partitionState;
             this.cd = cd;
             this.vds = vds;
             this.lts = lts;
@@ -504,22 +513,21 @@ public class Reconciler
 
         public String toString()
         {
-            return "RowState{" +
-                   "cd=" + (cd == STATIC_CLUSTERING ? "static" : cd) +
-                   ", vds=" + Arrays.toString(vds) +
-                   ", lts=" + Arrays.toString(lts) +
-                   '}';
+            return toString((SchemaSpec) null);
         }
 
         public String toString(SchemaSpec schema)
         {
-            return "RowState{" +
-                   "cd=" + (cd == STATIC_CLUSTERING ? "static" : cd) +
-                   ", vds=" + Arrays.toString(vds) +
-                   ", lts=" + Arrays.toString(lts) +
-                   ", clustering=" + (cd == STATIC_CLUSTERING ? "static" : Arrays.toString(schema.inflateClusteringKey(cd))) +
-                   ", values=" + Arrays.toString(cd == STATIC_CLUSTERING ? schema.inflateStaticColumns(vds) : schema.inflateRegularColumns(vds)) +
-                   '}';
+            return " rowStateRow("
+                   + partitionState.pd +
+                   "L, " + cd +
+                   (partitionState.staticRow == null ? "" : ", values(" + StringUtils.toString(partitionState.staticRow.vds) + ")") +
+                   (partitionState.staticRow == null ? "" : ", lts(" + StringUtils.toString(partitionState.staticRow.lts) + ")") +
+                   ", values(" + StringUtils.toString(vds) + ")" +
+                   ", lts(" + StringUtils.toString(lts) + ")" +
+                   (schema == null ? "" : ", clustering=" + (cd == STATIC_CLUSTERING ? "static" : Arrays.toString(schema.inflateClusteringKey(cd)))) +
+                   (schema == null ? "" : ", values=" + Arrays.toString(cd == STATIC_CLUSTERING ? schema.inflateStaticColumns(vds) : schema.inflateRegularColumns(vds))) +
+                   ")";
         }
     }
 }
\ No newline at end of file
diff --git a/harry-core/src/harry/runner/DataTracker.java b/harry-core/src/harry/runner/DataTracker.java
index 0171e43..82bd1dd 100644
--- a/harry-core/src/harry/runner/DataTracker.java
+++ b/harry-core/src/harry/runner/DataTracker.java
@@ -18,68 +18,49 @@
 
 package harry.runner;
 
-import java.util.ArrayList;
-import java.util.List;
 import java.util.function.LongConsumer;
 
 import harry.core.Configuration;
+import harry.ddl.SchemaSpec;
+import harry.model.OpSelectors;
 
-public abstract class DataTracker
+public interface DataTracker
 {
-    protected List<LongConsumer> onStarted = new ArrayList<>();
-    protected List<LongConsumer> onFinished = new ArrayList<>();
+    void onLtsStarted(LongConsumer onLts);
+    void onLtsFinished(LongConsumer onLts);
 
-    public void onLtsStarted(LongConsumer onLts)
-    {
-        this.onStarted.add(onLts);
-    }
-
-    public void onLtsFinished(LongConsumer onLts)
-    {
-        this.onFinished.add(onLts);
-    }
-
-    public void started(long lts)
-    {
-        startedInternal(lts);
-        for (LongConsumer consumer : onStarted)
-            consumer.accept(lts);
-    }
-
-    public void finished(long lts)
-    {
-        finishedInternal(lts);
-        for (LongConsumer consumer : onFinished)
-            consumer.accept(lts);
-    }
+    void beginModification(long lts);
+    void endModification(long lts);
 
-    abstract void startedInternal(long lts);
-    abstract void finishedInternal(long lts);
+    default void beginValidation(long pd) {}
+    default void endValidation(long pd) {}
 
-    public abstract long maxStarted();
-    public abstract long maxConsecutiveFinished();
+    long maxStarted();
+    boolean isFinished(long lts);
 
-    public abstract Configuration.DataTrackerConfiguration toConfig();
+    Configuration.DataTrackerConfiguration toConfig();
 
-    public static interface DataTrackerFactory
+    interface DataTrackerFactory
     {
-        DataTracker make();
+        DataTracker make(OpSelectors.PdSelector pdSelector, SchemaSpec schemaSpec);
     }
 
-    public static final DataTracker NO_OP = new NoOpDataTracker();
+    DataTracker NO_OP = new NoOpDataTracker();
 
-    public static class NoOpDataTracker extends DataTracker
+    class NoOpDataTracker implements DataTracker
     {
         private NoOpDataTracker() {}
 
-        protected void startedInternal(long lts) {}
-        protected void finishedInternal(long lts) {}
+        public void onLtsStarted(LongConsumer onLts){}
+        public void onLtsFinished(LongConsumer onLts){}
+
+        public void beginModification(long lts){}
+        public void endModification(long lts){}
+
         public long maxStarted() { return 0; }
-        public long maxConsecutiveFinished() { return 0; }
+        public boolean isFinished(long lts) { return false; }
 
-        public Configuration.DataTrackerConfiguration toConfig()
-        {
-            return null;
-        }
+        public Configuration.DataTrackerConfiguration toConfig(){ return null; }
     }
+
 }
diff --git a/harry-core/src/harry/runner/DefaultDataTracker.java b/harry-core/src/harry/runner/DefaultDataTracker.java
index 85e2637..bf05cd9 100644
--- a/harry-core/src/harry/runner/DefaultDataTracker.java
+++ b/harry-core/src/harry/runner/DefaultDataTracker.java
@@ -19,42 +19,71 @@
 package harry.runner;
 
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.PriorityBlockingQueue;
 import java.util.concurrent.atomic.AtomicLong;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.util.function.LongConsumer;
 
 import harry.core.Configuration;
 import harry.core.VisibleForTesting;
+import harry.concurrent.WaitQueue;
 
-public class DefaultDataTracker extends DataTracker
+public class DefaultDataTracker implements DataTracker
 {
-    private static final Logger logger = LoggerFactory.getLogger(DefaultDataTracker.class);
+     protected final AtomicLong maxSeenLts;
+    protected final AtomicLong maxCompleteLts;
+    protected final PriorityBlockingQueue<Long> reorderBuffer;
+    protected final DrainReorderQueueTask reorderTask;
 
-    private final AtomicLong maxSeenLts;
-    // TODO: This is a trivial implementation that can be significantly improved upon
-    // for example, we could use a bitmap that records `1`s for all lts that are after
-    // the consecutive, and "collapse" the bitmap state into the long as soon as we see
-    // consecutive `1` on the left side.
-    private final AtomicLong maxCompleteLts;
-    private final PriorityBlockingQueue<Long> reorderBuffer;
+    protected List<LongConsumer> onStarted = new ArrayList<>();
+    protected List<LongConsumer> onFinished = new ArrayList<>();
 
     public DefaultDataTracker()
     {
         this.maxSeenLts = new AtomicLong(-1);
         this.maxCompleteLts = new AtomicLong(-1);
         this.reorderBuffer = new PriorityBlockingQueue<>(100);
+        this.reorderTask = new DrainReorderQueueTask();
+        this.reorderTask.start();
+    }
+
+    @Override
+    public void onLtsStarted(LongConsumer onLts)
+    {
+        this.onStarted.add(onLts);
+    }
+
+    @Override
+    public void onLtsFinished(LongConsumer onLts)
+    {
+        this.onFinished.add(onLts);
     }
 
-    // TODO: there's also some room for improvement in terms of concurrency
-    protected void startedInternal(long lts)
+    @Override
+    public void beginModification(long lts)
+    {
+        assert lts >= 0;
+        startedInternal(lts);
+        for (LongConsumer consumer : onStarted)
+            consumer.accept(lts);
+    }
+
+    @Override
+    public void endModification(long lts)
+    {
+        finishedInternal(lts);
+        for (LongConsumer consumer : onFinished)
+            consumer.accept(lts);
+    }
+
+    void startedInternal(long lts)
     {
         recordEvent(lts, false);
     }
 
-    protected void finishedInternal(long lts)
+    void finishedInternal(long lts)
     {
         recordEvent(lts, true);
     }
@@ -67,44 +96,59 @@ public class DefaultDataTracker extends DataTracker
         if (!finished)
             return;
 
-        long maxAchievedConsecutive = drainReorderQueue();
-
-        if (maxAchievedConsecutive + 1 == lts)
-            maxCompleteLts.compareAndSet(maxAchievedConsecutive, lts);
-        else
+        if (!maxCompleteLts.compareAndSet(lts - 1, lts))
             reorderBuffer.offer(lts);
+
+        reorderTask.notify.signalAll();
     }
 
-    public long drainReorderQueue()
+    private class DrainReorderQueueTask extends Thread
     {
-        final long expected = maxCompleteLts.get();
-        long maxAchievedConsecutive = expected;
-        if (reorderBuffer.isEmpty())
-            return maxAchievedConsecutive;
-
-        boolean catchingUp = false;
+        private final WaitQueue notify;
 
-        Long smallest = reorderBuffer.poll();
-        while (smallest != null && smallest == maxAchievedConsecutive + 1)
+        private DrainReorderQueueTask()
         {
-            maxAchievedConsecutive++;
-            catchingUp = true;
-            smallest = reorderBuffer.poll();
+            super("DrainReorderQueueTask");
+            this.notify = WaitQueue.newWaitQueue();
         }
 
-        // put back
-        if (smallest != null)
-            reorderBuffer.offer(smallest);
-
-        if (catchingUp)
-            maxCompleteLts.compareAndSet(expected, maxAchievedConsecutive);
+        public void run()
+        {
+            while (!Thread.interrupted())
+            {
+                try
+                {
+                    WaitQueue.Signal signal = notify.register();
+                    runOnce();
+                    signal.awaitUninterruptibly();
+                }
+                catch (Throwable t)
+                {
+                    t.printStackTrace();
+                }
+            }
+        }
 
-        int bufferSize = reorderBuffer.size();
-        if (bufferSize > 100)
-            logger.warn("Reorder buffer size has grown up to " + reorderBuffer.size());
-        return maxAchievedConsecutive;
+        public void runOnce()
+        {
+            long maxAchievedConsecutive = maxCompleteLts.get();
+            if (reorderBuffer.isEmpty())
+                return;
+
+            Long smallest = reorderBuffer.peek();
+            while (smallest != null && smallest == maxAchievedConsecutive + 1)
+            {
+                boolean res = maxCompleteLts.compareAndSet(maxAchievedConsecutive, smallest);
+                assert res : String.format("Should have exclusive access to maxCompleteLts, but someone wrote %d, while %d was expected", maxCompleteLts.get(), maxAchievedConsecutive);
+                maxAchievedConsecutive = smallest;
+                long removed = reorderBuffer.remove();
+                assert smallest == removed : String.format("Tried to remove %d but removed %d", smallest, removed);
+                smallest = reorderBuffer.peek();
+            }
+        }
     }
 
+
     public long maxStarted()
     {
         return maxSeenLts.get();
@@ -112,22 +156,30 @@ public class DefaultDataTracker extends DataTracker
 
     public long maxConsecutiveFinished()
     {
-        if (!reorderBuffer.isEmpty())
-            return drainReorderQueue();
-
         return maxCompleteLts.get();
     }
 
+    public boolean isFinished(long lts)
+    {
+        return lts <= maxConsecutiveFinished() || reorderBuffer.contains(lts);
+    }
+
     public Configuration.DataTrackerConfiguration toConfig()
     {
-        return new Configuration.DefaultDataTrackerConfiguration(maxSeenLts.get(), maxCompleteLts.get());
+        return new Configuration.DefaultDataTrackerConfiguration(maxSeenLts.get(), maxCompleteLts.get(), new ArrayList<>(reorderBuffer));
     }
 
     @VisibleForTesting
-    public void forceLts(long maxSeen, long maxComplete)
+    public void forceLts(long maxSeen, long maxComplete, List<Long> reorderBuffer)
     {
+        System.out.printf("Forcing maxSeen: %d, maxComplete: %d, reorderBuffer: %s%n", maxSeen, maxComplete, reorderBuffer);
         this.maxSeenLts.set(maxSeen);
         this.maxCompleteLts.set(maxComplete);
+        if (reorderBuffer != null)
+        {
+            reorderBuffer.sort(Long::compareTo);
+            this.reorderBuffer.addAll(reorderBuffer);
+        }
     }
 
     public String toString()
diff --git a/harry-core/src/harry/runner/HarryRunner.java b/harry-core/src/harry/runner/HarryRunner.java
index 4dbfe76..32702bc 100644
--- a/harry-core/src/harry/runner/HarryRunner.java
+++ b/harry-core/src/harry/runner/HarryRunner.java
@@ -20,9 +20,6 @@ package harry.runner;
 
 import java.io.File;
 import java.io.FileNotFoundException;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -35,81 +32,37 @@ public abstract class HarryRunner
 {
     public static final Logger logger = LoggerFactory.getLogger(HarryRunner.class);
 
-    protected final boolean localRun = Boolean.parseBoolean(System.getProperty("harry.local-run", "true"));
-    protected final String cassandraVersion = System.getProperty("harry.cassandra-version");
-    protected final long startedAt = Long.getLong("harry.start-time", System.currentTimeMillis());
-
-    protected CompletableFuture<?> progress;
-    protected ScheduledThreadPoolExecutor executor;
-    public abstract void beforeRun(Runner.TimedRunner runner);
-    public void afterRun(Runner runner, Object result)
-    {
-        boolean success = !(result instanceof Throwable);
-
-        executor.shutdown();
-        try
-        {
-            executor.awaitTermination(10, TimeUnit.SECONDS);
-        }
-        catch (InterruptedException e)
-        {
-            // ignore
-        }
-    }
-
     public void run(Configuration config) throws Throwable
     {
         System.setProperty("cassandra.disable_tcactive_openssl", "true");
         System.setProperty("relocated.shaded.io.netty.transport.noNative", "true");
         System.setProperty("org.apache.cassandra.disable_mbean_registration", "true");
 
-        executor = new ScheduledThreadPoolExecutor(1);
-        executor.setRemoveOnCancelPolicy(true);
-
         Runner runner = config.createRunner();
         Run run = runner.getRun();
 
-        progress = runner.initAndStartAll();
-        
-        assert runner instanceof Runner.TimedRunner : "Please use a timed runner at the top level.";
-        beforeRun((Runner.TimedRunner) runner);
-
-        Object result = null;
+        Throwable thrown = null;
 
         try
         {
-            result = progress.handle((a, b) -> {
-                if (b != null)
-                    return b;
-                return a;
-            }).get();
-            if (result instanceof Throwable)
-                logger.error("Execution failed!", (Throwable) result);
-
+            beforeRun(runner);
+            runner.run();
         }
         catch (Throwable e)
         {
             logger.error("Failed due to exception: " + e.getMessage(), e);
-            result = e;
+            thrown = e;
         }
         finally
         {
-            afterRun(runner, result);
-
-            logger.info("Shutting down executor..");
-            tryRun(() -> {
-                executor.shutdownNow();
-                executor.awaitTermination(10, TimeUnit.SECONDS);
-            });
-
             logger.info("Shutting down runner..");
-            tryRun(runner::shutdown);
-            boolean failed = result instanceof Throwable;
+            boolean failed = thrown != null;
             if (!failed)
             {
                 logger.info("Shutting down cluster..");
                 tryRun(run.sut::shutdown);
             }
+            afterRun(runner, thrown);
             logger.info("Exiting...");
             if (failed)
                 System.exit(1);
@@ -136,7 +89,7 @@ public abstract class HarryRunner
      * @return Configuration YAML file.
      * @throws Exception If file is not found or cannot be read.
      */
-    public static File loadConfig(String[] args) throws Exception {
+    public static File loadConfig(String... args) throws Exception {
         if (args == null || args.length == 0) {
             throw new Exception("Harry config YAML not provided.");
         }
@@ -152,4 +105,7 @@ public abstract class HarryRunner
 
         return configFile;
     }
+
+    public abstract void beforeRun(Runner runner);
+    public abstract void afterRun(Runner runner, Object result);
 }
\ No newline at end of file
diff --git a/harry-core/src/harry/runner/LockingDataTracker.java b/harry-core/src/harry/runner/LockingDataTracker.java
new file mode 100644
index 0000000..a3d1968
--- /dev/null
+++ b/harry-core/src/harry/runner/LockingDataTracker.java
@@ -0,0 +1,257 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package harry.runner;
+
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
+
+import harry.concurrent.WaitQueue;
+import harry.core.Configuration;
+import harry.ddl.SchemaSpec;
+import harry.model.OpSelectors;
+
+/**
+ * Locking data tracker, that can be used with a quiescent model checker while providing
+ * a high degree of concurrency. It works by isolating readers from writers. In other words,
+ * readers can intersect with other readers, and writers can coincide with other writers.
+ *
+ * We achieve quiescence on a partition level, not on LTS level, and we know for sure
+ * which operations have finished for a partition, even if their LTS are non-contiguous.
+ *
+ * We use a simple wait queue for queuing up waiters, and a compact long counter for
+ * tracking the number of concurrent readers and writers. Lower 32 bits hold a number of
+ * readers, and higher 32 bits - a number of writers.
+ */
+public class LockingDataTracker extends DefaultDataTracker
+{
+    private final Map<Long, ReadersWritersLock> locked = new ConcurrentHashMap<>();
+
+    private final WaitQueue waitQueue = WaitQueue.newWaitQueue();
+    private final OpSelectors.PdSelector pdSelector;
+    private final SchemaSpec schemaSpec;
+
+    public LockingDataTracker(OpSelectors.PdSelector pdSelector, SchemaSpec schemaSpec)
+    {
+        this.pdSelector = pdSelector;
+        this.schemaSpec = schemaSpec;
+    }
+
+    @Override
+    public void beginModification(long lts)
+    {
+        ReadersWritersLock partitionLock = getLockForLts(lts);
+        partitionLock.lockForWrite();
+        super.beginModification(lts);
+    }
+
+    @Override
+    public void endModification(long lts)
+    {
+        super.endModification(lts);
+        getLockForLts(lts).unlockAfterWrite();
+    }
+
+    @Override
+    public void beginValidation(long pd)
+    {
+        ReadersWritersLock partitionLock = getLock(pd);
+        partitionLock.lockForRead();
+        super.beginValidation(pd);
+    }
+
+    @Override
+    public void endValidation(long pd)
+    {
+        super.endValidation(pd);
+        getLock(pd).unlockAfterRead();
+    }
+
+    public void validate(long pd, Runnable runnable)
+    {
+        ReadersWritersLock partitionLock = getLockForLts(pd);
+        partitionLock.lockForRead();
+        runnable.run();
+        partitionLock.unlockAfterRead();
+    }
+
+    private ReadersWritersLock getLockForLts(long lts)
+    {
+        long pd = pdSelector.pd(lts, schemaSpec);
+        return getLock(pd);
+    }
+
+    private ReadersWritersLock getLock(long pd)
+    {
+        return locked.computeIfAbsent(pd, (pd_) -> new ReadersWritersLock(waitQueue, pd));
+    }
+
+    /**
+     * Readers/writers lock. It was decided not to use signals here, and instead go for a
+     * busyspin instead, since we expect locks to be released briefly and contention to be minimal.
+     */
+    public static class ReadersWritersLock
+    {
+        private static final AtomicLongFieldUpdater<ReadersWritersLock> fieldUpdater = AtomicLongFieldUpdater.newUpdater(ReadersWritersLock.class, "lock");
+        private volatile long lock;
+
+        final long descriptor;
+        final WaitQueue waitQueue;
+
+        public ReadersWritersLock(WaitQueue waitQueue, long descriptor)
+        {
+            this.waitQueue = waitQueue;
+            this.lock = 0L;
+            this.descriptor = descriptor;
+        }
+
+        @Override
+        public String toString()
+        {
+            long lock = this.lock;
+            return "PartitionLock{" +
+                   "pd = " + descriptor +
+                   ", readers = " + getReaders(lock) +
+                   ", writers = " + getWriters(lock) +
+                   '}';
+        }
+
+        public void lockForWrite()
+        {
+            while (true)
+            {
+                WaitQueue.Signal signal = waitQueue.register();
+                long v = lock;
+                if (getReaders(v) == 0)
+                {
+                    if (fieldUpdater.compareAndSet(this, v, incWriters(v)))
+                    {
+                        signal.cancel();
+                        return;
+                    }
+                }
+                signal.awaitUninterruptibly();
+            }
+        }
+
+        public void unlockAfterWrite()
+        {
+            while (true)
+            {
+                long v = lock;
+                if (fieldUpdater.compareAndSet(this, v, decWriters(v)))
+                {
+                    waitQueue.signalAll();
+                    return;
+                }
+            }
+        }
+
+        public void lockForRead()
+        {
+            while (true)
+            {
+                WaitQueue.Signal signal = waitQueue.register();
+                long v = lock;
+                if (getWriters(v) == 0)
+                {
+                    if (fieldUpdater.compareAndSet(this, v, incReaders(v)))
+                    {
+                        signal.cancel();
+                        return;
+                    }
+                }
+                signal.awaitUninterruptibly();
+            }
+        }
+
+        public boolean tryLockForRead()
+        {
+            long v = lock;
+            if (getWriters(v) == 0 && fieldUpdater.compareAndSet(this, v, incReaders(v)))
+                return true;
+
+            return false;
+        }
+
+        public void unlockAfterRead()
+        {
+            while (true)
+            {
+                long v = lock;
+                if (fieldUpdater.compareAndSet(this, v, decReaders(v)))
+                {
+                    waitQueue.signalAll();
+                    return;
+                }
+            }
+        }
+
+        private long incReaders(long v)
+        {
+            long readers = getReaders(v);
+            v &= ~0xffffffffL; // erase all readers
+            return v | (readers + 1L);
+        }
+
+        private long decReaders(long v)
+        {
+            long readers = getReaders(v);
+            assert readers >= 1;
+            v &= ~0xffffffffL; // erase all readers
+            return v | (readers - 1L);
+        }
+
+        private long incWriters(long v)
+        {
+            long writers = getWriters(v);
+            v &= ~0xffffffff00000000L; // erase all writers
+            return v | ((writers + 1L) << 32);
+        }
+
+        private long decWriters(long v)
+        {
+            long writers = getWriters(v);
+            assert writers >= 1 : "Writers left " + writers;
+            v &= ~0xffffffff00000000L; // erase all writers
+            return v | ((writers - 1L) << 32);
+        }
+
+        public int getReaders(long v)
+        {
+            v &= 0xffffffffL;
+            return (int) v;
+        }
+
+        public int getWriters(long v)
+        {
+            v >>= 32;
+            v &= 0xffffffffL;
+            return (int) v;
+        }
+    }
+
+    @Override
+    public Configuration.DataTrackerConfiguration toConfig()
+    {
+        return new Configuration.LockingDataTrackerConfiguration(maxSeenLts.get(), maxCompleteLts.get(), new ArrayList<>(reorderBuffer));
+    }
+
+}
diff --git a/harry-core/src/harry/runner/Runner.java b/harry-core/src/harry/runner/Runner.java
index 207394d..1fc7592 100644
--- a/harry-core/src/harry/runner/Runner.java
+++ b/harry-core/src/harry/runner/Runner.java
@@ -24,57 +24,67 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.util.ArrayList;
-import java.util.Collection;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
-import java.util.concurrent.CompletableFuture;
+import java.util.Map;
 import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.BooleanSupplier;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import harry.concurrent.ExecutorFactory;
+import harry.concurrent.Interruptible;
+import harry.concurrent.Shutdownable;
+import harry.concurrent.WaitQueue;
 import harry.core.Configuration;
 import harry.core.Run;
 import harry.visitors.Visitor;
 
+import static harry.concurrent.Clock.Global.nanoTime;
+import static harry.concurrent.InfiniteLoopExecutor.Daemon.NON_DAEMON;
+import static harry.concurrent.InfiniteLoopExecutor.Interrupts.UNSYNCHRONIZED;
+import static harry.concurrent.InfiniteLoopExecutor.SimulatorSafe.SAFE;
+
 public abstract class Runner
 {
     private static final Logger logger = LoggerFactory.getLogger(Runner.class);
 
     protected final Run run;
     protected final Configuration config;
-    protected final ScheduledExecutorService executor;
 
     // If there's an error, there's a good chance we're going to hit it more than once
     // since we have multiple concurrent checkers running
-    protected final CopyOnWriteArrayList<Throwable> errors;
+    protected final List<Throwable> errors;
 
-    public Runner(Run run, Configuration config, int concurrency)
+    public Runner(Run run, Configuration config)
     {
         this.run = run;
         this.config = config;
         this.errors = new CopyOnWriteArrayList<>();
-        this.executor = Executors.newScheduledThreadPool(concurrency);
     }
 
-    public Run getRun()
+    public void run() throws Throwable
     {
-        return run;
+        init(config, run);
+        runInternal();
+        teardown();
     }
 
-    public int getErrorCount()
+    protected abstract void runInternal() throws Throwable;
+
+    public Run getRun()
     {
-        return errors.size();
+        return run;
     }
 
-    public void init()
+    public static void init(Configuration config, Run run)
     {
         if (config.create_schema)
         {
@@ -124,51 +134,8 @@ public abstract class Runner
         }
     }
 
-    public CompletableFuture<?> initAndStartAll()
-    {
-        init();
-        return start();
-    }
-
     public abstract String type();
         
-    public abstract void shutdown() throws InterruptedException;
-    
-    protected CompletableFuture<?> start()
-    {
-        return start(true, () -> false);
-    }
-    
-    protected abstract void shutDownVisitors();
-
-    @SuppressWarnings("ResultOfMethodCallIgnored")
-    protected void shutDownExecutors() throws InterruptedException
-    {
-        executor.shutdownNow();
-        executor.awaitTermination(1, TimeUnit.MINUTES);
-    }
-    
-    protected abstract CompletableFuture<?> start(boolean reportErrors, BooleanSupplier parentExit);
-
-    protected Runnable reportThrowable(Runnable runnable, CompletableFuture<?> future)
-    {
-        return () ->
-        {
-            try
-            {
-                if (!future.isDone())
-                    runnable.run();
-            }
-            catch (Throwable t)
-            {
-                errors.add(t);
-                
-                if (!future.isDone())
-                    future.completeExceptionally(t);
-            }
-        };
-    }
-
     public interface RunnerFactory
     {
         Runner make(Run run, Configuration config);
@@ -178,39 +145,19 @@ public abstract class Runner
     {
         public final long runtime;
         public final TimeUnit runtimeUnit;
-        
-        protected final ScheduledExecutorService shutdownExecutor;
 
-        public TimedRunner(Run run, Configuration config, int concurrency, long runtime, TimeUnit runtimeUnit)
+        public TimedRunner(Run run, Configuration config, long runtime, TimeUnit runtimeUnit)
         {
-            super(run, config, concurrency);
+            super(run, config);
 
-            this.shutdownExecutor = Executors.newSingleThreadScheduledExecutor();
             this.runtime = runtime;
             this.runtimeUnit = runtimeUnit;
         }
+    }
 
-        protected ScheduledFuture<?> scheduleTermination(AtomicBoolean terminated)
-        {
-            return shutdownExecutor.schedule(() ->
-                                             {
-                                                 logger.info("Runner has reached configured runtime. Stopping...");
-                                                 // TODO: wait for the last full validation?
-                                                 terminated.set(true);
-                                             },
-                                             runtime,
-                                             runtimeUnit);
-        }
-
-        @Override
-        @SuppressWarnings("ResultOfMethodCallIgnored")
-        protected void shutDownExecutors() throws InterruptedException
-        {
-            shutdownExecutor.shutdownNow();
-            shutdownExecutor.awaitTermination(1, TimeUnit.MINUTES);
-            executor.shutdownNow();
-            executor.awaitTermination(1, TimeUnit.MINUTES);
-        }
+    public static Configuration.RunnerConfiguration single(Visitor.VisitorFactory visitor)
+    {
+        return (run, config) -> new SingleVisitRunner(run, config, Collections.singletonList(visitor));
     }
 
     public static class SingleVisitRunner extends Runner
@@ -218,10 +165,10 @@ public abstract class Runner
         private final List<Visitor> visitors;
 
         public SingleVisitRunner(Run run,
-                                Configuration config,
-                                List<? extends Visitor.VisitorFactory> visitorFactories)
+                                 Configuration config,
+                                 List<? extends Visitor.VisitorFactory> visitorFactories)
         {
-            super(run, config, 1);
+            super(run, config);
             this.visitors = visitorFactories.stream().map(factory -> factory.make(run)).collect(Collectors.toList());
         }
 
@@ -231,50 +178,30 @@ public abstract class Runner
             return "single";
         }
 
-        @Override
-        protected CompletableFuture<?> start(boolean reportErrors, BooleanSupplier parentExit)
-        {
-            CompletableFuture<?> future = new CompletableFuture<>();
-
-            if (reportErrors)
-                future.whenComplete((a, b) -> maybeReportErrors());
-
-            executor.submit(reportThrowable(() -> run(visitors, future, parentExit), future));
-            return future;
-        }
-
-        private void run(List<Visitor> visitors, CompletableFuture<?> future, BooleanSupplier parentExit)
+        public void runInternal()
         {
-            for (Visitor value: visitors)
-            {
-                if (parentExit.getAsBoolean())
-                    break;
-
+            for (Visitor value : visitors)
                 value.visit();
-            }
-
-            future.complete(null);
         }
+    }
 
-        @Override
-        public void shutdown() throws InterruptedException
-        {
-            logger.info("Shutting down...");
-            shutDownVisitors();
-
-            // we need to wait for all threads that use schema to stop before we can tear down and drop the table
-            shutDownExecutors();
-
-            teardown();
-        }
+    public static Configuration.RunnerConfiguration sequential(Visitor.VisitorFactory visitor,
+                                                               long runtime, TimeUnit runtimeUnit)
+    {
+        return (r, c) -> new SequentialRunner(r, c, Collections.singletonList(visitor), runtime, runtimeUnit);
+    }
 
-        @Override
-        protected void shutDownVisitors()
-        {
-            shutDownVisitors(visitors);
-        }
+    public static Runner sequential(Run run,
+                                    Configuration config,
+                                    List<Visitor.VisitorFactory> poolConfigurations,
+                                    long runtime, TimeUnit runtimeUnit)
+    {
+        return new SequentialRunner(run, config, poolConfigurations, runtime, runtimeUnit);
     }
 
+    /**
+     * Runs all visitors sequentially, in the loop, for a specified amount of time
+     */
     public static class SequentialRunner extends TimedRunner
     {
         protected final List<Visitor> visitors;
@@ -284,7 +211,7 @@ public abstract class Runner
                                 List<? extends Visitor.VisitorFactory> visitorFactories,
                                 long runtime, TimeUnit runtimeUnit)
         {
-            super(run, config, 1, runtime, runtimeUnit);
+            super(run, config, runtime, runtimeUnit);
 
             this.visitors = visitorFactories.stream().map(factory -> factory.make(run)).collect(Collectors.toList());
         }
@@ -296,85 +223,38 @@ public abstract class Runner
         }
 
         @Override
-        protected CompletableFuture<?> start(boolean reportErrors, BooleanSupplier parentExit)
+        public void runInternal() throws Throwable
         {
-            CompletableFuture<?> future = new CompletableFuture<>();
-            
-            if (reportErrors)
-                future.whenComplete((a, b) -> maybeReportErrors());
-
-            AtomicBoolean terminated = new AtomicBoolean(false);
-            scheduleTermination(terminated);
-            BooleanSupplier exit = () -> Thread.currentThread().isInterrupted() || future.isDone() 
-                                         || terminated.get() || parentExit.getAsBoolean();
-
-            executor.submit(reportThrowable(() -> run(visitors, future, exit), future));
-            return future;
-        }
-
-        protected void run(List<Visitor> visitors,
-                           CompletableFuture<?> future,
-                           BooleanSupplier exit)
-        {
-            while (!exit.getAsBoolean())
+            long deadline = nanoTime() + runtimeUnit.toNanos(runtime);;
+            while (nanoTime() < deadline)
             {
-                for (Visitor visitor: visitors)
-                {
-                    if (exit.getAsBoolean())
-                        break;
+                for (Visitor visitor : visitors)
                     visitor.visit();
-                }
             }
-
-            future.complete(null);
-        }
-
-        @Override
-        public void shutdown() throws InterruptedException
-        {
-            logger.info("Shutting down...");
-
-            shutDownVisitors();
-
-            // we need to wait for all threads that use schema to stop before we can tear down and drop the table
-            shutDownExecutors();
-            
-            teardown();
         }
+    }
 
-        @Override
-        protected void shutDownVisitors()
-        {
-            shutDownVisitors(visitors);
-        }
+    public static Runner concurrent(Configuration config,
+                                    List<Configuration.VisitorPoolConfiguration> poolConfigurations,
+                                    long runtime, TimeUnit runtimeUnit)
+    {
+        return new ConcurrentRunner(config.createRun(), config, poolConfigurations, runtime, runtimeUnit);
     }
 
-    // TODO: this requires some significant improvement
+    /**
+     * Runs all visitors concurrently, each visitor in its own thread, looped, for a specified amount of time
+     */
     public static class ConcurrentRunner extends TimedRunner
     {
-        private final List<List<Visitor>> perThreadVisitors;
-        private final int concurrency;
+        private final List<Configuration.VisitorPoolConfiguration> poolConfigurations;
 
         public ConcurrentRunner(Run run,
                                 Configuration config,
-                                int concurrency,
-                                List<? extends Visitor.VisitorFactory> visitorFactories,
+                                List<Configuration.VisitorPoolConfiguration> poolConfigurations,
                                 long runtime, TimeUnit runtimeUnit)
         {
-            super(run, config, concurrency, runtime, runtimeUnit);
-
-            this.concurrency = concurrency;
-            this.perThreadVisitors = new ArrayList<>(concurrency);
-
-            for (int i = 0; i < concurrency; i++)
-            {
-                List<Visitor> visitors = new ArrayList<>();
-
-                for (Visitor.VisitorFactory factory : visitorFactories)
-                    visitors.add(factory.make(run));
-
-                perThreadVisitors.add(visitors);
-            }
+            super(run, config, runtime, runtimeUnit);
+            this.poolConfigurations = poolConfigurations;
         }
 
         @Override
@@ -383,93 +263,122 @@ public abstract class Runner
             return "concurrent";
         }
 
-        @Override
-        protected CompletableFuture<?> start(boolean reportErrors, BooleanSupplier parentExit)
+        public void runInternal() throws Throwable
         {
-            CompletableFuture<?> future = new CompletableFuture<>();
-            
-            if (reportErrors)
-                future.whenComplete((a, b) -> maybeReportErrors());
-
-            AtomicBoolean terminated = new AtomicBoolean(false);
-            scheduleTermination(terminated);
-            BooleanSupplier exit = () -> Thread.currentThread().isInterrupted() || future.isDone() 
-                                         || terminated.get() || parentExit.getAsBoolean();
-            
-            AtomicInteger liveCount = new AtomicInteger(0);
-            
-            for (int i = 0; i < concurrency; i++)
-            {
-                List<Visitor> visitors = perThreadVisitors.get(i);
-                executor.submit(reportThrowable(() -> run(visitors, future, exit, liveCount), future));
-            }
-
-            return future;
-        }
+            List<Interruptible> threads = new ArrayList<>();
+            WaitQueue queue = WaitQueue.newWaitQueue();
+            WaitQueue.Signal interrupt = queue.register();
 
-        private void run(List<Visitor> visitors,
-                         CompletableFuture<?> future,
-                         BooleanSupplier exit,
-                         AtomicInteger liveCount)
-        {
-            liveCount.incrementAndGet();
-            
-            while (!exit.getAsBoolean())
+            for (Configuration.VisitorPoolConfiguration poolConfiguration : poolConfigurations)
             {
-                for (Visitor visitor : visitors)
+                for (int i = 0; i < poolConfiguration.concurrency; i++)
                 {
-                    if (exit.getAsBoolean())
-                        break;
-
-                    visitor.visit();
+                    Visitor visitor = poolConfiguration.visitor.make(run);
+                    String name = String.format("%s-%d", poolConfiguration.prefix, i + 1);
+                    Interruptible thread = ExecutorFactory.Global.executorFactory().infiniteLoop(name, wrapInterrupt((state) -> {
+                        if (state == Interruptible.State.NORMAL)
+                            visitor.visit();
+                    }, interrupt::signal, errors::add), SAFE, NON_DAEMON, UNSYNCHRONIZED);
+                    threads.add(thread);
                 }
             }
-            
-            // If we're the last worker still running, complete the future....
-            if (liveCount.decrementAndGet() == 0)
-                future.complete(null);
+
+            interrupt.await(runtime, runtimeUnit);
+            shutdown(threads::stream);
+            if (!errors.isEmpty())
+                throw merge(errors);
         }
+    }
 
-        @Override
-        public void shutdown() throws InterruptedException
-        {
-            logger.info("Shutting down...");
-            shutDownVisitors();
+    public static Runner chain(Configuration config,
+                               Configuration.RunnerConfiguration... runners)
+    {
+        return new ChainRunner(config.createRun(), config, Arrays.asList(runners));
+    }
 
-            // we need to wait for all threads that use schema to stop before we can tear down and drop the table
-            shutDownExecutors();
+    /**
+     * Chain runner is somewhat similar to StagedRunner in that it would execute multiple runners one after
+     * another, but unlike StagedRunner, it is single-shot: it will just run them once.
+     */
+    public static class ChainRunner extends Runner
+    {
+        private final List<Runner> stages;
 
-            teardown();
+        public ChainRunner(Run run,
+                           Configuration config,
+                           List<Configuration.RunnerConfiguration> runnerFactories)
+        {
+            super(run, config);
+            this.stages = new ArrayList<>();
+            for (Configuration.RunnerConfiguration runner : runnerFactories)
+                stages.add(runner.make(run, config));
         }
 
-        @Override
-        protected void shutDownVisitors()
+        protected void runInternal() throws Throwable
         {
-            shutDownVisitors(perThreadVisitors.stream().flatMap(Collection::stream).collect(Collectors.toList()));
+            for (Runner stage : stages)
+                stage.runInternal();
+        }
+
+        public String type()
+        {
+            return "chain";
         }
     }
 
-    protected static void shutDownVisitors(List<Visitor> visitors)
+    public static void shutdown(Supplier<Stream<Interruptible>> threads)
     {
-        Throwable error = null;
-        
-        for (Visitor visitor : visitors)
-        {
+        threads.get().forEach(Shutdownable::shutdown);
+        threads.get().forEach((interruptible) -> {
             try
             {
-                visitor.shutdown();
+                if (!interruptible.awaitTermination(60, TimeUnit.SECONDS))
+                    logger.info("Could not terminate before the timeout: " + threads.get().map(Shutdownable::isTerminated).collect(Collectors.toList()));
             }
             catch (InterruptedException e)
             {
-                if (error != null)
-                    error.addSuppressed(e);
-                else
-                    error = e;
+                throw new RuntimeException(e);
             }
+        });
+
+    }
+
+    public static Interruptible.Task wrapInterrupt(Interruptible.Task runnable, Runnable signalStop, Consumer<Throwable> registerException)
+    {
+        return (state) -> {
+            try
+            {
+                runnable.run(state);
+            }
+            catch (InterruptedException t)
+            {
+                signalStop.run();
+            }
+            catch (Throwable t)
+            {
+                // Since some of the exceptions are thrown from inside instances
+                if (!t.getClass().toString().contains("InterruptedException"))
+                    registerException.accept(t);
+                signalStop.run();
+            }
+        };
+    }
+
+    public static Throwable merge(List<Throwable> existingFail)
+    {
+        if (existingFail.size() == 1)
+            return existingFail.get(0);
+
+        Throwable e = existingFail.get(0);
+        Throwable ret = e;
+        for (int i = 1; i < existingFail.size(); i++)
+        {
+            Throwable current = existingFail.get(i);
+            e.addSuppressed(current);
+            e = current;
         }
-        
-        if (error != null)
-            logger.warn("Failed to shut down all visitors!", error);
+
+        return ret;
     }
 
     private static void dumpExceptionToFile(BufferedWriter bw, Throwable throwable) throws IOException
@@ -496,7 +405,7 @@ public abstract class Runner
         bw.newLine();
     }
 
-    private static void dumpStateToFile(Run run, Configuration config, List<Throwable> t)
+    public static void dumpStateToFile(Run run, Configuration config, List<Throwable> t)
     {
         try
         {
@@ -518,12 +427,9 @@ public abstract class Runner
             builder.setClock(run.clock.toConfig());
             builder.setDataTracker(run.tracker.toConfig());
 
-            try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file))))
-            {
-                bw.write(Configuration.toYamlString(builder.build()));
-                bw.flush();
-            }
+            Configuration.toFile(file, builder.build());
         }
+
         catch (Throwable e)
         {
             logger.error("Caught an error while trying to dump to file", e);
diff --git a/harry-core/src/harry/runner/StagedRunner.java b/harry-core/src/harry/runner/StagedRunner.java
index 88ebcdb..4e61f1e 100644
--- a/harry-core/src/harry/runner/StagedRunner.java
+++ b/harry-core/src/harry/runner/StagedRunner.java
@@ -18,34 +18,22 @@
 
 package harry.runner;
 
+import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.BooleanSupplier;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import harry.core.Configuration;
 import harry.core.Run;
 
+import static harry.concurrent.Clock.Global.nanoTime;
+
 public class StagedRunner extends Runner.TimedRunner
 {
     public static final String TYPE = "staged";
 
-    public static void register()
-    {
-        Configuration.registerSubtypes(StagedRunner.StagedRunnerConfig.class);
-    }
-    
-    private static final Logger logger = LoggerFactory.getLogger(StagedRunner.class);
-    
-    private final List<Configuration.RunnerConfiguration> runnerFactories;
     private final List<Runner> stages;
 
     public StagedRunner(Run run,
@@ -53,9 +41,10 @@ public class StagedRunner extends Runner.TimedRunner
                         List<Configuration.RunnerConfiguration> runnerFactories,
                         long runtime, TimeUnit runtimeUnit)
     {
-        super(run, config, 1, runtime, runtimeUnit);
-        this.runnerFactories = runnerFactories;
-        this.stages = new CopyOnWriteArrayList<>();
+        super(run, config, runtime, runtimeUnit);
+        this.stages = new ArrayList<>();
+        for (Configuration.RunnerConfiguration runner : runnerFactories)
+            stages.add(runner.make(run, config));
     }
 
     @Override
@@ -64,112 +53,14 @@ public class StagedRunner extends Runner.TimedRunner
     }
 
     @Override
-    protected CompletableFuture<?> start(boolean reportErrors, BooleanSupplier parentExit)
+    protected void runInternal() throws Throwable
     {
-        CompletableFuture<?> future = new CompletableFuture<>();
-
-        if (reportErrors)
-        {
-            future.whenComplete((a, b) ->
-            {
-                collectErrors();
-                maybeReportErrors();
-            });
-        }
-
-        AtomicBoolean terminated = new AtomicBoolean(false);
-        scheduleTermination(terminated);
-        
-        BooleanSupplier exit = () -> Thread.currentThread().isInterrupted() || future.isDone()
-                                     || terminated.get() || parentExit.getAsBoolean();
-
-        executor.submit(() ->
-        {
-            for (Configuration.RunnerConfiguration runnerConfig : runnerFactories)
-            {
-                try
-                {
-                    Runner runner = runnerConfig.make(run, config);
-                    if (runner instanceof StagedRunner)
-                        throw new IllegalArgumentException("StagedRunner can not be nested inside of a StagedRunner");
-                    stages.add(runner);
-                }
-                catch (Throwable t)
-                {
-                    future.completeExceptionally(t);
-                }
-            }
-
-            while (!exit.getAsBoolean())
-            {
-                logger.info("Starting next staged run...");
-                int stage = 1;
-
-                for (Runner runner : stages)
-                {
-                    if (exit.getAsBoolean())
-                        break;
-
-                    try
-                    {
-                        logger.info("Starting stage {}: {}...", stage++, runner.type());
-                        runner.start(false, exit).get();
-
-                        if (!terminated.get())
-                            logger.info("...stage complete.");
-
-                        // Wait for any previous runners to settle down...
-                        while (run.tracker.maxConsecutiveFinished() != run.tracker.maxStarted())
-                        {
-                            TimeUnit.SECONDS.sleep(1);
-                            logger.warn("Waiting for any previous runners to settle down: {}",
-                                        run.tracker.toString());
-                        }
-
-                    }
-                    catch (Throwable t)
-                    {
-                        future.completeExceptionally(t);
-                        break;
-                    }
-                }
-
-                if (!terminated.get())
-                    logger.info("All stages complete!");
-            }
+        long deadline = nanoTime() + runtimeUnit.toNanos(nanoTime());
 
-            future.complete(null);
-        });
-
-        return future;
-    }
-
-    @Override
-    public void shutdown() throws InterruptedException
-    {
-        logger.info("Shutting down...");
-
-        for (Runner runner : stages)
-        {
-            runner.shutDownVisitors();
-            runner.shutDownExecutors();
-        }
-
-        shutDownExecutors();
-        teardown();
-    }
-
-    @Override
-    protected void shutDownVisitors()
-    {
-        // Visitors are shut down in the sub-runners.    
-    }
-
-    private void collectErrors()
-    {
-        for (Runner runner : stages)
+        while (nanoTime() < deadline)
         {
-            errors.addAll(runner.errors);
+            for (Runner runner : stages)
+                runner.runInternal();
         }
     }
 
diff --git a/harry-integration/src/harry/runner/TrivialShrinker.java b/harry-core/src/harry/runner/TrivialShrinker.java
similarity index 98%
rename from harry-integration/src/harry/runner/TrivialShrinker.java
rename to harry-core/src/harry/runner/TrivialShrinker.java
index b598b85..d82c0b6 100644
--- a/harry-integration/src/harry/runner/TrivialShrinker.java
+++ b/harry-core/src/harry/runner/TrivialShrinker.java
@@ -92,7 +92,7 @@ public class TrivialShrinker
                 if (pdsToSkip.contains(pdToCheck))
                     continue;
                 pdsToSkip.add(pdToCheck);
-                config.make(run, configuration).init();
+                Runner.init(configuration, run);
 
                 try
                 {
@@ -125,7 +125,7 @@ public class TrivialShrinker
                 if (ltsToSkip.contains(ltsToCheck) || pdsToSkip.contains(run.pdSelector.pd(lts, run.schemaSpec)))
                     continue;
                 ltsToSkip.add(ltsToCheck);
-                config.make(run, configuration).init();
+                Runner.init(configuration, run);
 
                 try
                 {
diff --git a/harry-core/src/harry/runner/UpToLtsRunner.java b/harry-core/src/harry/runner/UpToLtsRunner.java
index f89ecbb..f34df45 100644
--- a/harry-core/src/harry/runner/UpToLtsRunner.java
+++ b/harry-core/src/harry/runner/UpToLtsRunner.java
@@ -18,11 +18,9 @@
 
 package harry.runner;
 
+import java.util.Collections;
 import java.util.List;
-import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.BooleanSupplier;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -31,16 +29,31 @@ import harry.core.Configuration;
 import harry.core.Run;
 import harry.visitors.Visitor;
 
+import static harry.concurrent.Clock.Global.nanoTime;
+
+/**
+ * Runner that allows to run for a specific number of logical timestamps, rather
+ * than being configured by time.
+ */
 public class UpToLtsRunner extends Runner.SequentialRunner
 {
     public static final String TYPE = "up_to_lts";
 
-    public static void register()
+    private final long maxLts;
+
+    public static Configuration.RunnerConfiguration factory(List<? extends Visitor.VisitorFactory> visitorFactories,
+                                                            long maxLts,
+                                                            long runtime, TimeUnit runtimeUnit)
     {
-        Configuration.registerSubtypes(UpToLtsRunnerConfig.class);
+        return (r, c) -> new UpToLtsRunner(r, c, visitorFactories, maxLts, runtime, runtimeUnit);
     }
 
-    private final long maxLts;
+    public static Configuration.RunnerConfiguration factory(Visitor.VisitorFactory visitorFactory,
+                                                            long maxLts,
+                                                            long runtime, TimeUnit runtimeUnit)
+    {
+        return (r, c) -> new UpToLtsRunner(r, c, Collections.singletonList(visitorFactory), maxLts, runtime, runtimeUnit);
+    }
 
     public UpToLtsRunner(Run run,
                          Configuration config, 
@@ -59,21 +72,14 @@ public class UpToLtsRunner extends Runner.SequentialRunner
     }
 
     @Override
-    protected CompletableFuture<?> start(boolean reportErrors, BooleanSupplier parentExit)
+    public void runInternal()
     {
-        CompletableFuture<?> future = new CompletableFuture<>();
-
-        if (reportErrors)
-            future.whenComplete((a, b) -> maybeReportErrors());
-
-        AtomicBoolean terminated = new AtomicBoolean(false);
-        scheduleTermination(terminated);
-        BooleanSupplier exit = () -> run.clock.peek() >= maxLts
-                                     || Thread.currentThread().isInterrupted() || future.isDone()
-                                     || terminated.get() || parentExit.getAsBoolean();
-
-        executor.submit(reportThrowable(() -> run(visitors, future, exit), future));
-        return future;
+        long deadline = nanoTime() + runtimeUnit.toNanos(runtime);
+        while (run.tracker.maxStarted() < maxLts && nanoTime() < deadline)
+        {
+            for (Visitor visitor : visitors)
+                visitor.visit();
+        }
     }
 
     @JsonTypeName(TYPE)
diff --git a/harry-core/src/harry/util/ByteUtils.java b/harry-core/src/harry/util/ByteUtils.java
index 4adf536..17153a3 100644
--- a/harry-core/src/harry/util/ByteUtils.java
+++ b/harry-core/src/harry/util/ByteUtils.java
@@ -92,6 +92,13 @@ public class ByteUtils
         return b;
     }
 
+    public static ByteBuffer[] objectsToBytes(Object... objects)
+    {
+        ByteBuffer[] bytes = new ByteBuffer[objects.length];
+        for (int i = 0; i < objects.length; i++)
+            bytes[i] = objectToBytes(objects[i]);
+        return bytes;
+    }
 
     public static ByteBuffer objectToBytes(Object obj)
     {
@@ -131,46 +138,6 @@ public class ByteUtils
                                                              obj.getClass()));
     }
 
-    public static ByteBuffer pack(Collection<ByteBuffer> buffers, int elements)
-    {
-        int size = 0;
-        for (ByteBuffer bb : buffers)
-            size += sizeOfValue(bb);
-
-        ByteBuffer result = ByteBuffer.allocate(sizeOfCollectionSize(elements) + size);
-        writeCollectionSize(result, elements);
-        for (ByteBuffer bb : buffers)
-            writeValue(result, bb);
-        result.flip();
-        return result;
-    }
-
-    public static int sizeOfValue(ByteBuffer value)
-    {
-        return value == null ? 4 : 4 + value.remaining();
-    }
-
-    protected static void writeCollectionSize(ByteBuffer output, int elements)
-    {
-        output.putInt(elements);
-    }
-    public static void writeValue(ByteBuffer output, ByteBuffer value)
-    {
-        if (value == null)
-        {
-            output.putInt(-1);
-            return;
-        }
-
-        output.putInt(value.remaining());
-        output.put(value.duplicate());
-    }
-
-    protected static int sizeOfCollectionSize(int elements)
-    {
-        return 4;
-    }
-
     public static ByteBuffer compose(ByteBuffer... buffers) {
         if (buffers.length == 1) return buffers[0];
 
diff --git a/harry-integration/test/harry/model/TestBaseImpl.java b/harry-core/src/harry/util/StringUtils.java
similarity index 72%
copy from harry-integration/test/harry/model/TestBaseImpl.java
copy to harry-core/src/harry/util/StringUtils.java
index 2a91086..45a0b6a 100644
--- a/harry-integration/test/harry/model/TestBaseImpl.java
+++ b/harry-core/src/harry/util/StringUtils.java
@@ -16,17 +16,20 @@
  *  limitations under the License.
  */
 
-package harry.model;
+package harry.util;
 
-import org.junit.BeforeClass;
-
-import org.apache.cassandra.distributed.api.ICluster;
-
-// TODO: probably redundant
-public class TestBaseImpl extends org.apache.cassandra.distributed.test.TestBaseImpl
+public class StringUtils
 {
-    @BeforeClass
-    public static void beforeClass() throws Throwable {
-        ICluster.setup();
+    public static String toString(long[] arr)
+    {
+        String s = "";
+        for (int i = 0; i < arr.length; i++)
+        {
+            s += arr[i];
+            s += "L";
+            if (i < (arr.length - 1))
+                s += ',';
+        }
+        return s;
     }
 }
diff --git a/harry-core/src/harry/visitors/AllPartitionsValidator.java b/harry-core/src/harry/visitors/AllPartitionsValidator.java
index 7bd5b50..be37d34 100644
--- a/harry-core/src/harry/visitors/AllPartitionsValidator.java
+++ b/harry-core/src/harry/visitors/AllPartitionsValidator.java
@@ -18,16 +18,19 @@
 
 package harry.visitors;
 
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.LongPredicate;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import harry.concurrent.ExecutorFactory;
+import harry.concurrent.Interruptible;
+import harry.concurrent.WaitQueue;
+import harry.core.Configuration;
 import harry.core.MetricReporter;
 import harry.core.Run;
 import harry.ddl.SchemaSpec;
@@ -35,9 +38,16 @@ import harry.model.Model;
 import harry.model.OpSelectors;
 import harry.model.sut.SystemUnderTest;
 import harry.operations.Query;
+import harry.runner.DataTracker;
+import harry.runner.Runner;
 
-// This might be something that potentially grows into the validator described in the design doc;
-// right now it's just a helper/container class
+import static harry.concurrent.InfiniteLoopExecutor.Daemon.NON_DAEMON;
+import static harry.concurrent.InfiniteLoopExecutor.Interrupts.UNSYNCHRONIZED;
+import static harry.concurrent.InfiniteLoopExecutor.SimulatorSafe.SAFE;
+
+/**
+ * Concurrently validates all partitions that were visited during this run.
+ */
 public class AllPartitionsValidator implements Visitor
 {
     private static final Logger logger = LoggerFactory.getLogger(AllPartitionsValidator.class);
@@ -48,21 +58,32 @@ public class AllPartitionsValidator implements Visitor
     protected final OpSelectors.MonotonicClock clock;
     protected final OpSelectors.PdSelector pdSelector;
     protected final MetricReporter metricReporter;
-    protected final ExecutorService executor;
     protected final SystemUnderTest sut;
+    protected final DataTracker tracker;
 
-    protected final AtomicBoolean condition;
-    protected final AtomicLong maxPos;
-
+    protected final LongPredicate condition;
     protected final int concurrency;
-    protected final int triggerAfter;
 
-    public AllPartitionsValidator(int concurrency,
-                                  int triggerAfter,
-                                  Run run,
+    public static Configuration.VisitorConfiguration factory(int concurrency,
+                                                             LongPredicate condition,
+                                                             Model.ModelFactory modelFactory)
+    {
+        return (r) -> new AllPartitionsValidator(r, concurrency, condition, modelFactory);
+    }
+
+    public AllPartitionsValidator(Run run,
+                                  int concurrency,
+                                  long triggerAfter,
+                                  Model.ModelFactory modelFactory)
+    {
+        this(run, concurrency, (lts) -> (lts > 0 && lts % triggerAfter == 0), modelFactory);
+    }
+
+    public AllPartitionsValidator(Run run,
+                                  int concurrency,
+                                  LongPredicate condition,
                                   Model.ModelFactory modelFactory)
     {
-        this.triggerAfter = triggerAfter;
         this.metricReporter = run.metricReporter;
         this.model = modelFactory.make(run);
         this.schema = run.schemaSpec;
@@ -70,68 +91,57 @@ public class AllPartitionsValidator implements Visitor
         this.sut = run.sut;
         this.pdSelector = run.pdSelector;
         this.concurrency = concurrency;
-        this.executor = Executors.newFixedThreadPool(concurrency);
-        this.condition = new AtomicBoolean();
-        this.maxPos = new AtomicLong(-1);
-        run.tracker.onLtsStarted((lts) -> {
-            maxPos.updateAndGet(current -> Math.max(pdSelector.positionFor(lts), current));
-            if (triggerAfter == 0 || (triggerAfter > 0 && lts % triggerAfter == 0))
-                condition.set(true);
-        });
+        this.tracker = run.tracker;
+        this.condition = condition;
     }
 
-    protected CompletableFuture<Void> validateAllPartitions(ExecutorService executor, int parallelism)
+    protected void validateAllPartitions() throws Throwable
     {
-        final long maxPos = this.maxPos.get();
-        AtomicLong cnt = new AtomicLong();
-        CompletableFuture<?>[] futures = new CompletableFuture[parallelism];
-        AtomicBoolean isDone = new AtomicBoolean(false);
-
-        for (int i = 0; i < parallelism; i++)
+        List<Interruptible> threads = new ArrayList<>();
+        WaitQueue queue = WaitQueue.newWaitQueue();
+        WaitQueue.Signal interrupt = queue.register();
+        List<Throwable> errors = new CopyOnWriteArrayList<>();
+
+        final long maxPosition = pdSelector.maxPosition(tracker.maxStarted());
+        AtomicLong currentPosition = new AtomicLong();
+        for (int i = 0; i < concurrency; i++)
         {
-            futures[i] = CompletableFuture.supplyAsync(() -> {
-                long pos;
-                while ((pos = cnt.getAndIncrement()) < maxPos && !executor.isShutdown() && !Thread.interrupted() && !isDone.get())
-                {
-                    if (pos > 0 && pos % 100 == 0)
-                        logger.info(String.format("Validated %d out of %d partitions", pos, maxPos));
-                    long visitLts = pdSelector.minLtsAt(pos);
-
-                    metricReporter.validatePartition();
-
-                    for (boolean reverse : new boolean[]{ true, false })
-                    {
-                        try
-                        {
-                            model.validate(Query.selectPartition(schema, pdSelector.pd(visitLts, schema), reverse));
-                        }
-                        catch (Throwable t)
-                        {
-                            isDone.set(true);
-                            logger.error("Caught an error while validating all partitions.", t);
-                            throw t;
-                        }
-                    }
-                }
-                return null;
-            }, executor);
+            Interruptible thread = ExecutorFactory.Global.executorFactory().infiniteLoop(String.format("AllPartitionsValidator-%d", i + 1),
+                                                                                         Runner.wrapInterrupt((state) -> {
+                                                                                             if (state == Interruptible.State.NORMAL)
+                                                                                             {
+                                                                                                 metricReporter.validatePartition();
+                                                                                                 long pos = currentPosition.getAndIncrement();
+                                                                                                 if (pos < maxPosition)
+                                                                                                 {
+                                                                                                     for (boolean reverse : new boolean[]{ true, false })
+                                                                                                         model.validate(Query.selectPartition(schema, pdSelector.pd(pdSelector.minLtsAt(pos), schema), reverse));
+                                                                                                 }
+                                                                                                 else
+                                                                                                 {
+                                                                                                     interrupt.signalAll();
+                                                                                                 }
+                                                                                             }
+                                                                                         }, interrupt::signal, errors::add), SAFE, NON_DAEMON, UNSYNCHRONIZED);
+            threads.add(thread);
         }
 
-        return CompletableFuture.allOf(futures);
+        interrupt.awaitUninterruptibly();
+
+        Runner.shutdown(threads::stream);
+        if (!errors.isEmpty())
+            throw Runner.merge(errors);
     }
 
     public void visit()
     {
-        // TODO: this is ok for now, but if/when we bring exhaustive checker back, we need to change this:
-        // we'll probably want a low-concurrency validator somewhere in background all the time.
-        if (condition.compareAndSet(true, false))
+        long lts = clock.peek();
+        if (condition.test(lts))
         {
-            long lts = clock.peek();
             logger.info("Starting validation of all partitions as of lts {}...", lts);
-
             try
             {
-                validateAllPartitions(executor, concurrency).get();
+                validateAllPartitions();
             }
             catch (Throwable e)
             {
@@ -142,11 +152,5 @@ public class AllPartitionsValidator implements Visitor
         }
     }
 
-    @SuppressWarnings("ResultOfMethodCallIgnored")
-    @Override
-    public void shutdown() throws InterruptedException
-    {
-        executor.shutdown();
-        executor.awaitTermination(60, TimeUnit.SECONDS);
-    }
+
 }
\ No newline at end of file
diff --git a/harry-core/src/harry/visitors/CorruptingVisitor.java b/harry-core/src/harry/visitors/CorruptingVisitor.java
index 4677ea2..e8a028e 100644
--- a/harry-core/src/harry/visitors/CorruptingVisitor.java
+++ b/harry-core/src/harry/visitors/CorruptingVisitor.java
@@ -52,6 +52,7 @@ public class CorruptingVisitor implements Visitor
                                                                 HideRowCorruptor::new),
         new AddExtraRowCorruptor(run.schemaSpec,
                                  run.clock,
+                                 run.tracker,
                                  run.descriptorSelector),
         new QueryResponseCorruptor.SimpleQueryResponseCorruptor(run.schemaSpec,
                                                                 run.clock,
diff --git a/harry-integration/src/harry/runner/FaultInjectingVisitor.java b/harry-core/src/harry/visitors/FaultInjectingVisitor.java
similarity index 93%
rename from harry-integration/src/harry/runner/FaultInjectingVisitor.java
rename to harry-core/src/harry/visitors/FaultInjectingVisitor.java
index 4cca1b2..3885c4e 100644
--- a/harry-integration/src/harry/runner/FaultInjectingVisitor.java
+++ b/harry-core/src/harry/visitors/FaultInjectingVisitor.java
@@ -16,7 +16,7 @@
  *  limitations under the License.
  */
 
-package harry.runner;
+package harry.visitors;
 
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executors;
@@ -31,33 +31,14 @@ import harry.core.Configuration;
 import harry.core.Run;
 import harry.model.sut.SystemUnderTest;
 import harry.operations.CompiledStatement;
-import harry.visitors.LoggingVisitor;
-import harry.visitors.OperationExecutor;
-import harry.visitors.Visitor;
 
+/**
+ * Fault injecting visitor: randomly fails some of the queries.
+ *
+ * Requires {@code FaultInjectingSut} to function.
+ */
 public class FaultInjectingVisitor extends LoggingVisitor
 {
-    public static void init()
-    {
-        Configuration.registerSubtypes(FaultInjectingVisitorConfiguration.class);
-    }
-
-    @JsonTypeName("fault_injecting")
-    public static class FaultInjectingVisitorConfiguration extends Configuration.MutatingVisitorConfiguation
-    {
-        @JsonCreator
-        public FaultInjectingVisitorConfiguration(@JsonProperty("row_visitor") Configuration.RowVisitorConfiguration row_visitor)
-        {
-            super(row_visitor);
-        }
-
-        @Override
-        public Visitor make(Run run)
-        {
-            return new FaultInjectingVisitor(run, row_visitor);
-        }
-    }
-
     private final AtomicInteger cnt = new AtomicInteger();
 
     private final SystemUnderTest.FaultInjectingSut sut;
@@ -96,4 +77,21 @@ public class FaultInjectingVisitor extends LoggingVisitor
                    originator.complete(res);
            });
     }
+
+    @JsonTypeName("fault_injecting")
+    public static class FaultInjectingVisitorConfiguration extends Configuration.MutatingVisitorConfiguation
+    {
+        @JsonCreator
+        public FaultInjectingVisitorConfiguration(@JsonProperty("row_visitor") Configuration.RowVisitorConfiguration row_visitor)
+        {
+            super(row_visitor);
+        }
+
+        @Override
+        public Visitor make(Run run)
+        {
+            return new FaultInjectingVisitor(run, row_visitor);
+        }
+    }
+
 }
diff --git a/harry-core/src/harry/visitors/MutatingVisitor.java b/harry-core/src/harry/visitors/MutatingVisitor.java
index db27216..c8f45f7 100644
--- a/harry-core/src/harry/visitors/MutatingVisitor.java
+++ b/harry-core/src/harry/visitors/MutatingVisitor.java
@@ -25,10 +25,12 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import harry.core.Configuration;
 import harry.core.Run;
 import harry.model.OpSelectors;
 import harry.model.sut.SystemUnderTest;
@@ -39,12 +41,27 @@ public class MutatingVisitor extends GeneratingVisitor
 {
     private static final Logger logger = LoggerFactory.getLogger(MutatingVisitor.class);
 
+    public MutatingVisitor(Run run)
+    {
+        this(run, MutatingRowVisitor::new);
+    }
+
     public MutatingVisitor(Run run,
                            OperationExecutor.RowVisitorFactory rowVisitorFactory)
     {
         this(run, new MutatingVisitExecutor(run, rowVisitorFactory.make(run)));
     }
 
+    public static Configuration.VisitorConfiguration factory()
+    {
+        return MutatingVisitor::new;
+    }
+
+    public static Configuration.VisitorConfiguration factory(Function<Run, VisitExecutor> rowVisitorFactory)
+    {
+        return (r) -> new MutatingVisitor(r, rowVisitorFactory.apply(r));
+    }
+
     public MutatingVisitor(Run run,
                            VisitExecutor visitExecutor)
     {
@@ -64,25 +81,33 @@ public class MutatingVisitor extends GeneratingVisitor
         protected final DataTracker tracker;
         protected final SystemUnderTest sut;
         protected final OperationExecutor rowVisitor;
+        protected final SystemUnderTest.ConsistencyLevel consistencyLevel;
         private final int maxRetries = 10;
 
         public MutatingVisitExecutor(Run run, OperationExecutor rowVisitor)
+        {
+            this(run, rowVisitor, SystemUnderTest.ConsistencyLevel.QUORUM);
+        }
+
+        public MutatingVisitExecutor(Run run, OperationExecutor rowVisitor, SystemUnderTest.ConsistencyLevel consistencyLevel)
         {
             this.descriptorSelector = run.descriptorSelector;
             this.tracker = run.tracker;
             this.sut = run.sut;
             this.rowVisitor = rowVisitor;
+            this.consistencyLevel = consistencyLevel;
         }
 
         @Override
         public void beforeLts(long lts, long pd)
         {
-            tracker.started(lts);
+            tracker.beginModification(lts);
         }
 
         @Override
         public void afterLts(long lts, long pd)
         {
+            // TODO: switch to Cassandra futures!
             for (CompletableFuture<?> future : futures)
             {
                 try
@@ -91,11 +116,15 @@ public class MutatingVisitor extends GeneratingVisitor
                 }
                 catch (Throwable t)
                 {
-                    throw new IllegalStateException("Couldn't repeat operations within timeout bounds.", t);
+                    int complete = 0;
+                    for (CompletableFuture<?> f : futures)
+                        if (f.isDone()) complete++;
+
+                    throw new IllegalStateException(String.format("Couldn't repeat operations within timeout bounds. %d out of %d futures complete", complete, futures.size()), t);
                 }
             }
             futures.clear();
-            tracker.finished(lts);
+            tracker.endModification(lts);
         }
 
         @Override
@@ -157,7 +186,7 @@ public class MutatingVisitor extends GeneratingVisitor
             if (retries > this.maxRetries)
                 throw new IllegalStateException(String.format("Can not execute statement %s after %d retries", statement, retries));
 
-            sut.executeAsync(statement.cql(), SystemUnderTest.ConsistencyLevel.QUORUM, statement.bindings())
+            sut.executeAsync(statement.cql(), consistencyLevel, statement.bindings())
                .whenComplete((res, t) -> {
                    if (t != null)
                    {
@@ -165,7 +194,8 @@ public class MutatingVisitor extends GeneratingVisitor
                        int delaySecs = 1;
                        executor.schedule(() -> executeAsyncWithRetries(future, statement, retries + 1), delaySecs, TimeUnit.SECONDS);
                        logger.info("Scheduled retry to happen with delay {} seconds", delaySecs);
-                   }else
+                   }
+                   else
                        future.complete(res);
                });
         }
diff --git a/harry-core/src/harry/visitors/RandomPartitionValidator.java b/harry-core/src/harry/visitors/RandomPartitionValidator.java
new file mode 100644
index 0000000..16cef9b
--- /dev/null
+++ b/harry-core/src/harry/visitors/RandomPartitionValidator.java
@@ -0,0 +1,61 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package harry.visitors;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import harry.core.Configuration;
+import harry.core.Run;
+import harry.model.Model;
+import harry.model.OpSelectors;
+import harry.operations.QueryGenerator;
+
+public class RandomPartitionValidator implements Visitor
+{
+    protected final Model model;
+    protected final QueryGenerator.TypedQueryGenerator queryGenerator;
+    protected final Run run;
+    protected final AtomicLong modifier;
+
+    public RandomPartitionValidator(Run run)
+    {
+        this(run, new Configuration.QuiescentCheckerConfig());
+    }
+
+    public RandomPartitionValidator(Run run,
+                                    Model.ModelFactory modelFactory)
+    {
+        this.model = modelFactory.make(run);
+        this.queryGenerator = new QueryGenerator.TypedQueryGenerator(run);
+        this.run = run;
+        this.modifier = new AtomicLong();
+    }
+
+    public void visit()
+    {
+        if (run.tracker.maxStarted() == 0)
+            return;
+
+        long modifier = this.modifier.incrementAndGet();
+        long pd = ((OpSelectors.DefaultPdSelector)run.pdSelector).randomVisitedPd(run.tracker.maxStarted(),
+                                                                                  modifier,
+                                                                                  run.schemaSpec);
+        model.validate(queryGenerator.inflate(run.pdSelector.maxLtsFor(pd), modifier));
+    }
+}
diff --git a/harry-integration/src/harry/visitors/SkippingVisitor.java b/harry-core/src/harry/visitors/SkippingVisitor.java
similarity index 100%
rename from harry-integration/src/harry/visitors/SkippingVisitor.java
rename to harry-core/src/harry/visitors/SkippingVisitor.java
diff --git a/harry-core/test/harry/operations/RelationTest.java b/harry-core/test/harry/operations/RelationTest.java
index eaa7ddb..178bd96 100644
--- a/harry-core/test/harry/operations/RelationTest.java
+++ b/harry-core/test/harry/operations/RelationTest.java
@@ -130,6 +130,11 @@ public class RelationTest
                                                                       {
                                                                           throw new RuntimeException("not implemented");
                                                                       }
+
+                                                                      public long maxPosition(long maxLts)
+                                                                      {
+                                                                          throw new RuntimeException("not implemented");
+                                                                      }
                                                                   },
                                                                   new OpSelectors.DescriptorSelector()
                                                                   {
diff --git a/harry-core/test/harry/runner/LockingDataTrackerTest.java b/harry-core/test/harry/runner/LockingDataTrackerTest.java
new file mode 100644
index 0000000..9cc5ce4
--- /dev/null
+++ b/harry-core/test/harry/runner/LockingDataTrackerTest.java
@@ -0,0 +1,120 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package harry.runner;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.LockSupport;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import harry.concurrent.ExecutorFactory;
+import harry.concurrent.Interruptible;
+import harry.concurrent.WaitQueue;
+import harry.ddl.SchemaGenerators;
+import harry.ddl.SchemaSpec;
+import harry.model.OpSelectors;
+
+import static harry.concurrent.InfiniteLoopExecutor.Daemon.NON_DAEMON;
+import static harry.concurrent.InfiniteLoopExecutor.Interrupts.UNSYNCHRONIZED;
+import static harry.concurrent.InfiniteLoopExecutor.SimulatorSafe.SAFE;
+import static harry.runner.Runner.merge;
+
+public class LockingDataTrackerTest
+{
+    @Test
+    public void testDataTracker() throws Throwable
+    {
+        SchemaSpec schemaSpec = SchemaGenerators.defaultSchemaSpecGen("test").inflate(1L);
+        OpSelectors.Rng rng = new OpSelectors.PCGFast(1L);
+        OpSelectors.PdSelector pdSelector = new OpSelectors.DefaultPdSelector(rng, 5, 2);
+        LockingDataTracker tracker = new LockingDataTracker(pdSelector, schemaSpec);
+
+        AtomicReference<State> excluded = new AtomicReference<>(State.UNLOCKED);
+        AtomicInteger readers = new AtomicInteger(0);
+        AtomicInteger writers = new AtomicInteger(0);
+
+        WaitQueue queue = WaitQueue.newWaitQueue();
+        WaitQueue.Signal interrupt = queue.register();
+        List<Throwable> errors = new CopyOnWriteArrayList<>();
+
+        long lts = 1;
+        long pd = pdSelector.pd(lts, schemaSpec);
+        for (int i = 0; i < 2; i++)
+        {
+            ExecutorFactory.Global.executorFactory().infiniteLoop("read-" + i, Runner.wrapInterrupt(state -> {
+                if (state == Interruptible.State.NORMAL)
+                {
+                    tracker.beginModification(lts);
+                    writers.incrementAndGet();
+                    Assert.assertEquals(0, readers.get());
+                    excluded.updateAndGet((prev) -> {
+                        assert (prev == State.UNLOCKED || prev == State.LOCKED_FOR_WRITE) : prev;
+                        return State.LOCKED_FOR_WRITE;
+                    });
+                    Assert.assertEquals(0, readers.get());
+                    excluded.updateAndGet((prev) -> {
+                        assert (prev == State.UNLOCKED || prev == State.LOCKED_FOR_WRITE) : prev;
+                        return State.UNLOCKED;
+                    });
+                    Assert.assertEquals(0, readers.get());
+                    writers.decrementAndGet();
+                    tracker.endModification(lts);
+                    LockSupport.parkNanos(100);
+                }
+            }, interrupt::signal, errors::add), SAFE, NON_DAEMON, UNSYNCHRONIZED);
+        }
+
+        for (int i = 0; i < 2; i++)
+        {
+            ExecutorFactory.Global.executorFactory().infiniteLoop("write-" + i, Runner.wrapInterrupt(state -> {
+                if (state == Interruptible.State.NORMAL)
+                {
+                    tracker.beginValidation(pd);
+                    readers.incrementAndGet();
+                    Assert.assertEquals(0, writers.get());
+                    excluded.updateAndGet((prev) -> {
+                        assert (prev == State.UNLOCKED || prev == State.LOCKED_FOR_READ) : prev;
+                        return State.LOCKED_FOR_READ;
+                    });
+                    Assert.assertEquals(0, writers.get());
+                    excluded.updateAndGet((prev) -> {
+                        assert (prev == State.UNLOCKED || prev == State.LOCKED_FOR_READ) : prev;
+                        return State.UNLOCKED;
+                    });
+                    Assert.assertEquals(0, writers.get());
+                    readers.decrementAndGet();
+                    tracker.endValidation(pd);
+                    LockSupport.parkNanos(100);
+                }
+            }, interrupt::signal, errors::add), SAFE, NON_DAEMON, UNSYNCHRONIZED);
+        }
+
+        interrupt.await(1, TimeUnit.MINUTES);
+        if (!errors.isEmpty())
+            throw merge(errors);
+    }
+    enum State { UNLOCKED, LOCKED_FOR_READ, LOCKED_FOR_WRITE }
+}
diff --git a/harry-integration-external/dependency-reduced-pom.xml b/harry-integration-external/dependency-reduced-pom.xml
index e2e00a5..23431b0 100644
--- a/harry-integration-external/dependency-reduced-pom.xml
+++ b/harry-integration-external/dependency-reduced-pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <artifactId>harry-parent</artifactId>
     <groupId>org.apache.cassandra</groupId>
-    <version>0.0.1-SNAPSHOT</version>
+    <version>0.0.2-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>harry-integration-external</artifactId>
diff --git a/harry-integration-external/pom.xml b/harry-integration-external/pom.xml
index 9592b43..e0d4528 100755
--- a/harry-integration-external/pom.xml
+++ b/harry-integration-external/pom.xml
@@ -40,7 +40,7 @@
         <dependency>
             <groupId>org.apache.cassandra</groupId>
             <artifactId>harry-core</artifactId>
-	        <version>${project.parent.version}</version>
+            <version>${project.parent.version}</version>
         </dependency>
 
         <dependency>
diff --git a/harry-integration/test/harry/model/TestBaseImpl.java b/harry-integration-external/src/com/datastax/driver/core/UtilsAccessor.java
similarity index 72%
rename from harry-integration/test/harry/model/TestBaseImpl.java
rename to harry-integration-external/src/com/datastax/driver/core/UtilsAccessor.java
index 2a91086..e22d125 100644
--- a/harry-integration/test/harry/model/TestBaseImpl.java
+++ b/harry-integration-external/src/com/datastax/driver/core/UtilsAccessor.java
@@ -16,17 +16,14 @@
  *  limitations under the License.
  */
 
-package harry.model;
+package com.datastax.driver.core;
 
-import org.junit.BeforeClass;
+import java.nio.ByteBuffer;
 
-import org.apache.cassandra.distributed.api.ICluster;
-
-// TODO: probably redundant
-public class TestBaseImpl extends org.apache.cassandra.distributed.test.TestBaseImpl
+public class UtilsAccessor
 {
-    @BeforeClass
-    public static void beforeClass() throws Throwable {
-        ICluster.setup();
+    public static long token(ByteBuffer bb)
+    {
+        return (long) com.datastax.driver.core.Token.getFactory("Murmur3Partitioner").hash(bb).getValue();
     }
 }
diff --git a/harry-integration-external/src/harry/model/sut/external/ExternalClusterSut.java b/harry-integration-external/src/harry/model/sut/external/ExternalClusterSut.java
index 76a4e3c..ae2808c 100644
--- a/harry-integration-external/src/harry/model/sut/external/ExternalClusterSut.java
+++ b/harry-integration-external/src/harry/model/sut/external/ExternalClusterSut.java
@@ -25,21 +25,24 @@ import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.util.concurrent.FutureCallback;
 import com.google.common.util.concurrent.Futures;
 import harry.core.Configuration;
+import harry.ddl.SchemaSpec;
 import harry.model.sut.SystemUnderTest;
+import harry.model.sut.TokenPlacementModel;
+import harry.util.ByteUtils;
 
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
 
 public class ExternalClusterSut implements SystemUnderTest
 {
-    public static void registerSubtypes()
-    {
-        Configuration.registerSubtypes(ExternalSutConfiguration.class);
-    }
-
     private final Session session;
     private final ExecutorService executor;
 
@@ -54,6 +57,11 @@ public class ExternalClusterSut implements SystemUnderTest
         this.executor = Executors.newFixedThreadPool(threads);
     }
 
+    public Session session()
+    {
+        return session;
+    }
+
     public static ExternalClusterSut create(ExternalSutConfiguration config)
     {
         // TODO: close Cluster and Session!
@@ -74,10 +82,10 @@ public class ExternalClusterSut implements SystemUnderTest
     public void shutdown()
     {
         session.close();
-        executor.shutdown();
+        executor.shutdownNow();
         try
         {
-            executor.awaitTermination(60, TimeUnit.SECONDS);
+            executor.awaitTermination(10, TimeUnit.SECONDS);
         }
         catch (InterruptedException e)
         {
@@ -107,8 +115,40 @@ public class ExternalClusterSut implements SystemUnderTest
         }
     }
 
+    public Object[][] executeNodeLocal(String statement, Predicate<Host> selector, Object... bindings)
+    {
+        int repeat = 10;
+        while (true)
+        {
+            try
+            {
+                Statement st = new SimpleStatement(statement, bindings);
+                boolean selected = false;
+                for (Host host : session.getCluster().getMetadata().getAllHosts())
+                {
+                    if (selector.test(host))
+                    {
+                        st.setHost(host);
+                        selected = true;
+                        break;
+                    }
+                }
+                assert selected;
+                return resultSetToObjectArray(session.execute(st));
+            }
+            catch (Throwable t)
+            {
+                if (repeat < 0)
+                    throw t;
+
+                t.printStackTrace();
+                repeat--;
+                // retry unconditionally
+            }
+        }
+    }
 
-    private static Object[][] resultSetToObjectArray(ResultSet rs)
+    public static Object[][] resultSetToObjectArray(ResultSet rs)
     {
         List<Row> rows = rs.all();
         if (rows.size() == 0)
diff --git a/harry-integration-external/src/harry/model/sut/external/QuiescentLocalStateChecker.java b/harry-integration-external/src/harry/model/sut/external/QuiescentLocalStateChecker.java
new file mode 100644
index 0000000..bf4f949
--- /dev/null
+++ b/harry-integration-external/src/harry/model/sut/external/QuiescentLocalStateChecker.java
@@ -0,0 +1,80 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package harry.model.sut.external;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NavigableMap;
+
+import com.datastax.driver.core.Host;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.UtilsAccessor;
+import harry.core.Run;
+import harry.model.QuiescentLocalStateCheckerBase;
+import harry.model.sut.TokenPlacementModel;
+import harry.util.ByteUtils;
+
+import static harry.model.sut.TokenPlacementModel.Node;
+
+public class QuiescentLocalStateChecker extends QuiescentLocalStateCheckerBase
+{
+    public QuiescentLocalStateChecker(Run run)
+    {
+        this(run, 3);
+    }
+
+    public QuiescentLocalStateChecker(Run run, int rf)
+    {
+        super(run, rf);
+    }
+
+    @Override
+    protected NavigableMap<TokenPlacementModel.Range, List<TokenPlacementModel.Node>> getRing()
+    {
+        Session session = ((ExternalClusterSut) sut).session();
+        Host control = session.getState().getConnectedHosts().stream().findFirst().get();
+        SimpleStatement peers = new SimpleStatement("select peer, tokens from system.peers");
+        peers.setHost(control);
+        List<TokenPlacementModel.Node> other = TokenPlacementModel.peerStateToNodes(ExternalClusterSut.resultSetToObjectArray(session.execute(peers)));
+        SimpleStatement local = new SimpleStatement("select broadcast_address, tokens from system.local");
+        local.setHost(control);
+
+        List<TokenPlacementModel.Node> self = TokenPlacementModel.peerStateToNodes(ExternalClusterSut.resultSetToObjectArray(session.execute(local)));
+        List<TokenPlacementModel.Node> all = new ArrayList<>();
+        all.addAll(self);
+        all.addAll(other);
+        return TokenPlacementModel.replicate(all, rf);
+    }
+
+    @Override
+    protected long token(long pd)
+    {
+        return UtilsAccessor.token(ByteUtils.compose(ByteUtils.objectsToBytes(schema.inflatePartitionKey(pd))));
+    }
+
+
+    @Override
+    protected Object[][] executeNodeLocal(String statement, Node node, Object... bindings)
+    {
+        return ((ExternalClusterSut) sut).executeNodeLocal(statement,
+                                                           (host) -> host.getAddress().toString().contains(node.id),
+                                                           bindings);
+    }
+}
diff --git a/harry-integration-external/src/harry/runner/external/HarryRunnerExternal.java b/harry-integration-external/src/harry/runner/external/HarryRunnerExternal.java
index 621d579..3252860 100644
--- a/harry-integration-external/src/harry/runner/external/HarryRunnerExternal.java
+++ b/harry-integration-external/src/harry/runner/external/HarryRunnerExternal.java
@@ -28,8 +28,6 @@ import java.io.File;
 public class HarryRunnerExternal extends HarryRunner {
 
     public static void main(String[] args) throws Throwable {
-        ExternalClusterSut.registerSubtypes();
-
         HarryRunner runner = new HarryRunnerExternal();
         File configFile = runner.loadConfig(args);
 
@@ -37,8 +35,11 @@ public class HarryRunnerExternal extends HarryRunner {
         runner.run(configuration);
     }
 
-    @Override
-    public void beforeRun(Runner.TimedRunner runner) {
+    public void beforeRun(Runner runner)
+    {
+    }
 
+    public void afterRun(Runner runner, Object result)
+    {
     }
 }
diff --git a/harry-integration-external/src/harry/runner/external/MiniStress.java b/harry-integration-external/src/harry/runner/external/MiniStress.java
new file mode 100644
index 0000000..5ad0574
--- /dev/null
+++ b/harry-integration-external/src/harry/runner/external/MiniStress.java
@@ -0,0 +1,126 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package harry.runner.external;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import harry.core.Configuration;
+import harry.core.Run;
+import harry.model.Model;
+import harry.model.OpSelectors;
+import harry.model.QuiescentChecker;
+import harry.model.sut.external.QuiescentLocalStateChecker;
+import harry.operations.Query;
+import harry.runner.HarryRunner;
+import harry.runner.LockingDataTracker;
+import harry.runner.Runner;
+import harry.visitors.AllPartitionsValidator;
+import harry.visitors.MutatingVisitor;
+import harry.visitors.Visitor;
+
+public class MiniStress
+{
+    public static void main(String... args) throws Throwable
+    {
+        File configFile = HarryRunner.loadConfig(args);
+        Configuration config = Configuration.fromFile(configFile);
+
+        List<Configuration.VisitorPoolConfiguration> concurrentPools = new ArrayList<>();
+        List<Visitor.VisitorFactory> postRunValidation = new ArrayList<>();
+        int duration = 60_000;
+        for (String arg : args)
+        {
+            if (arg.startsWith("--write"))
+            {
+                int concurrency = Integer.parseInt(arg.split("=")[1]);
+                concurrentPools.add(new Configuration.VisitorPoolConfiguration("Writer", concurrency, MutatingVisitor::new));
+            }
+            else if (arg.startsWith("--read"))
+            {
+                int concurrency = Integer.parseInt(arg.split("=")[1]);
+                AtomicLong modifier = new AtomicLong(0);
+                concurrentPools.add(new Configuration.VisitorPoolConfiguration("Reader", concurrency, (r) -> {
+                    Model model = new QuiescentChecker(r);
+                    OpSelectors.DefaultPdSelector pdSelector = (OpSelectors.DefaultPdSelector) r.pdSelector;
+
+                    return () -> {
+                        // Wait until we have something to validate
+                        if (r.tracker.maxStarted() == 0)
+                            return;
+
+                        long randomPd = pdSelector.randomVisitedPd(r.tracker.maxStarted(), modifier.incrementAndGet(), r.schemaSpec);
+                        Query query = Query.selectPartition(r.schemaSpec, randomPd, true);
+                        model.validate(query);
+                        query = Query.selectPartition(r.schemaSpec, randomPd, false);
+                        model.validate(query);
+                    };
+                }));
+            }
+            else if (arg.startsWith("--duration"))
+            {
+                duration = Integer.parseInt(arg.split("=")[1]);
+            }
+            else if (arg.startsWith("--validate-all-local"))
+            {
+                postRunValidation.add((r) -> new AllPartitionsValidator(r, 10, (lts) -> true, QuiescentLocalStateChecker::new));
+            }
+            else if (arg.startsWith("--validate-all"))
+            {
+                postRunValidation.add((r) -> new AllPartitionsValidator(r, 10, (lts) -> true, QuiescentChecker::new));
+            }
+        }
+
+        Configuration.ConfigurationBuilder builder = config.unbuild()
+                                                           .setRunner(null);
+        config = builder.build();
+        Run run = config.createRun();
+
+        if (!(run.tracker instanceof LockingDataTracker))
+        {
+            System.err.println("Concurrent read/write workloads only work with a locking data tracker.");
+            System.exit(1);
+        }
+
+        System.out.println(Configuration.toYamlString(config));
+
+        int finalDuration = duration;
+        new Runner.ChainRunner(run, config,
+                               Arrays.asList((r, c) -> new Runner.ConcurrentRunner(r, c, concurrentPools, finalDuration, TimeUnit.MILLISECONDS),
+                                             (r, c) -> new Runner.SingleVisitRunner(r, c, postRunValidation)))
+        .run();
+
+        run.sut.shutdown();
+
+        Configuration.toFile(new File(String.format("resume-%d.yaml", config.seed)),
+                             config.unbuild()
+                                   .setClock(run.clock.toConfig())
+                                   .setDataTracker(run.tracker.toConfig())
+                                   .setDropSchema(false)
+                                   .setTruncateTable(false)
+                                   .setCreateSchema(false)
+                                   .build());
+        System.exit(0);
+    }
+
+}
\ No newline at end of file
diff --git a/harry-integration/pom.xml b/harry-integration/pom.xml
index 0531e2d..03daedc 100755
--- a/harry-integration/pom.xml
+++ b/harry-integration/pom.xml
@@ -65,6 +65,10 @@
             <version>0.25</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+        </dependency>
     </dependencies>
     <build>
         <testResources>
diff --git a/harry-integration/src/harry/model/sut/ByteUtils.java b/harry-integration/src/harry/model/sut/ByteUtils.java
deleted file mode 100644
index ee7721c..0000000
--- a/harry-integration/src/harry/model/sut/ByteUtils.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- *  Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package harry.model.sut;
-
-import java.net.InetAddress;
-import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.AsciiType;
-import org.apache.cassandra.db.marshal.BooleanType;
-import org.apache.cassandra.db.marshal.ByteType;
-import org.apache.cassandra.db.marshal.DoubleType;
-import org.apache.cassandra.db.marshal.FloatType;
-import org.apache.cassandra.db.marshal.InetAddressType;
-import org.apache.cassandra.db.marshal.IntegerType;
-import org.apache.cassandra.db.marshal.ListType;
-import org.apache.cassandra.db.marshal.SetType;
-import org.apache.cassandra.db.marshal.ShortType;
-import org.apache.cassandra.db.marshal.UUIDType;
-
-public class ByteUtils extends harry.util.ByteUtils
-{
-    public static ByteBuffer objectToBytes(Object obj)
-    {
-        if (obj instanceof Integer)
-            return bytes((int) obj);
-        else if (obj instanceof Byte)
-            return bytes((byte) obj);
-        else if (obj instanceof Boolean)
-            return bytes( (byte) (((boolean) obj) ? 1 : 0));
-        else if (obj instanceof Short)
-            return bytes((short) obj);
-        else if (obj instanceof Long)
-            return bytes((long) obj);
-        else if (obj instanceof Float)
-            return bytes((float) obj);
-        else if (obj instanceof Double)
-            return bytes((double) obj);
-        else if (obj instanceof UUID)
-            return bytes((UUID) obj);
-        else if (obj instanceof InetAddress)
-            return bytes((InetAddress) obj);
-        else if (obj instanceof String)
-            return bytes((String) obj);
-        else if (obj instanceof List)
-        {
-            List l = (List) obj;
-            if (l.isEmpty())
-                return pack(Collections.emptyList(), 0);
-            return ListType.getInstance(getType(l.get(0)), true).decompose(l);
-        }
-        else if (obj instanceof Set)
-        {
-            Set l = (Set) obj;
-            if (l.isEmpty())
-                return pack(Collections.emptyList(), 0);
-            return SetType.getInstance(getType(l.iterator().next()), true).decompose(l);
-        }
-        else if (obj instanceof ByteBuffer)
-            return (ByteBuffer) obj;
-
-        else
-            throw new IllegalArgumentException(String.format("Cannot convert value %s of type %s",
-                                                             obj,
-                                                             obj.getClass()));
-    }
-
-    public static AbstractType<?> getType(Object obj)
-    {
-        if (obj instanceof Integer)
-            return IntegerType.instance; //TODO
-        else if (obj instanceof Byte)
-            return ByteType.instance;
-        else if (obj instanceof Boolean)
-            return BooleanType.instance;
-        else if (obj instanceof Short)
-            return ShortType.instance;
-        else if (obj instanceof Long)
-            return IntegerType.instance;
-        else if (obj instanceof Float)
-            return FloatType.instance;
-        else if (obj instanceof Double)
-            return DoubleType.instance;
-        else if (obj instanceof UUID)
-            return UUIDType.instance;
-        else if (obj instanceof InetAddress)
-            return InetAddressType.instance;
-        else if (obj instanceof String)
-            return AsciiType.instance;
-        else
-            throw new IllegalArgumentException(String.format("Cannot convert value %s of type %s",
-                                                             obj,
-                                                             obj.getClass()));
-    }
-}
diff --git a/harry-integration/src/harry/model/sut/InJvmSut.java b/harry-integration/src/harry/model/sut/injvm/InJvmSutConfiguration.java
similarity index 50%
rename from harry-integration/src/harry/model/sut/InJvmSut.java
rename to harry-integration/src/harry/model/sut/injvm/InJvmSutConfiguration.java
index 687a5b3..fdb33f9 100644
--- a/harry-integration/src/harry/model/sut/InJvmSut.java
+++ b/harry-integration/src/harry/model/sut/injvm/InJvmSutConfiguration.java
@@ -16,43 +16,22 @@
  *  limitations under the License.
  */
 
-package harry.model.sut;
+package harry.model.sut.injvm;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.function.Consumer;
-import java.util.stream.Collectors;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
-import harry.core.Configuration;
 import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ICluster;
 import org.apache.cassandra.distributed.api.IInstanceConfig;
 import org.apache.cassandra.distributed.api.IInvokableInstance;
-import org.apache.cassandra.locator.EndpointsForToken;
-import org.apache.cassandra.service.StorageService;
 
-public class InJvmSut extends InJvmSutBase<IInvokableInstance, Cluster>
-{
-    public static void init()
-    {
-        Configuration.registerSubtypes(InJvmSutConfiguration.class);
-    }
-
-    public InJvmSut(Cluster cluster)
-    {
-        super(cluster, 10);
-    }
-
-    public InJvmSut(Cluster cluster, int threads)
-    {
-        super(cluster, threads);
-    }
-
-    @JsonTypeName("in_jvm")
-    public static class InJvmSutConfiguration extends InJvmSutBaseConfiguration<IInvokableInstance, Cluster>
+@JsonTypeName("in_jvm")
+public class InJvmSutConfiguration extends InJvmSutBase.InJvmSutBaseConfiguration<IInvokableInstance, ICluster<IInvokableInstance>>
     {
         @JsonCreator
         public InJvmSutConfiguration(@JsonProperty(value = "nodes", defaultValue = "3") int nodes,
@@ -62,13 +41,13 @@ public class InJvmSut extends InJvmSutBase<IInvokableInstance, Cluster>
             super(nodes, worker_threads, root);
         }
 
-        protected Cluster cluster(Consumer<IInstanceConfig> cfg, int nodes, File root)
+        protected ICluster<IInvokableInstance> cluster(Consumer<IInstanceConfig> cfg, int nodes, File root)
         {
             try
             {
                 return Cluster.build().withConfig(cfg)
-                               .withNodes(nodes)
-                               .withRoot(root)
+                              .withNodes(nodes)
+                              .withRoot(root)
                               .createWithoutStarting();
             }
             catch (IOException e)
@@ -77,22 +56,8 @@ public class InJvmSut extends InJvmSutBase<IInvokableInstance, Cluster>
             }
         }
 
-        protected InJvmSutBase<IInvokableInstance, Cluster> sut(Cluster cluster)
+        protected InJvmSutBase<IInvokableInstance, ICluster<IInvokableInstance>> sut(ICluster<IInvokableInstance> cluster)
         {
             return new InJvmSut(cluster);
         }
-    }
-
-    public int[] getReplicasFor(Object[] partitionKey, String keyspace, String table)
-    {
-        return cluster.get(1).appliesOnInstance((Object[] pk, String ks) ->
-                                                {
-                                                    String pkString = Arrays.stream(pk).map(Object::toString).collect(Collectors.joining(":"));
-                                                    EndpointsForToken endpoints = StorageService.instance.getNaturalReplicasForToken(ks, table, pkString);
-                                                    int[] nodes = new int[endpoints.size()];
-                                                    for (int i = 0; i < endpoints.size(); i++)
-                                                        nodes[i] = endpoints.get(i).endpoint().address.getAddress()[3];
-                                                    return nodes;
-                                                }).apply(partitionKey, keyspace);
-    }
-}
+    }
\ No newline at end of file
diff --git a/harry-integration/src/harry/model/sut/MixedVersionInJvmSut.java b/harry-integration/src/harry/model/sut/injvm/MixedInJvmSutConfiguration.java
similarity index 60%
rename from harry-integration/src/harry/model/sut/MixedVersionInJvmSut.java
rename to harry-integration/src/harry/model/sut/injvm/MixedInJvmSutConfiguration.java
index 5ac03a7..dbead71 100644
--- a/harry-integration/src/harry/model/sut/MixedVersionInJvmSut.java
+++ b/harry-integration/src/harry/model/sut/injvm/MixedInJvmSutConfiguration.java
@@ -16,7 +16,8 @@
  *  limitations under the License.
  */
 
-package harry.model.sut;
+package harry.model.sut.injvm;
+
 
 import java.io.File;
 import java.io.IOException;
@@ -24,41 +25,17 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.function.Consumer;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
-import harry.core.Configuration;
 import org.apache.cassandra.distributed.UpgradeableCluster;
+import org.apache.cassandra.distributed.api.ICluster;
 import org.apache.cassandra.distributed.api.IInstanceConfig;
 import org.apache.cassandra.distributed.api.IUpgradeableInstance;
 import org.apache.cassandra.distributed.shared.Versions;
 
-/**
- * Make sure to set -Dcassandra.test.dtest_jar_path when using this class
- */
-public class MixedVersionInJvmSut extends InJvmSutBase<IUpgradeableInstance, UpgradeableCluster>
-{
-    public static void init()
-    {
-        Configuration.registerSubtypes(MixedInJvmSutConfiguration.class);
-    }
-
-    private static final Logger logger = LoggerFactory.getLogger(MixedVersionInJvmSut.class);
-    private final Versions.Version initialVersion;
-    private final List<Versions.Version> versions;
-
-    public MixedVersionInJvmSut(UpgradeableCluster cluster, Versions.Version initialVersion, List<Versions.Version> versions)
-    {
-        super(cluster, 10);
-        this.initialVersion = initialVersion;
-        this.versions = versions;
-    }
-
-    @JsonTypeName("mixed_in_jvm")
-    public static class MixedInJvmSutConfiguration extends InJvmSutBaseConfiguration<IUpgradeableInstance, UpgradeableCluster>
+@JsonTypeName("mixed_in_jvm")
+public class MixedInJvmSutConfiguration extends InJvmSutBase.InJvmSutBaseConfiguration<IUpgradeableInstance, ICluster<IUpgradeableInstance>>
     {
         public final String initial_version;
         public final List<String> versions;
@@ -85,7 +62,7 @@ public class MixedVersionInJvmSut extends InJvmSutBase<IUpgradeableInstance, Upg
                 upgradeVersions.add(allVersions.get(version));
         }
 
-        protected UpgradeableCluster cluster(Consumer<IInstanceConfig> cfg, int nodes, File root)
+        protected ICluster<IUpgradeableInstance> cluster(Consumer<IInstanceConfig> cfg, int nodes, File root)
         {
             try
             {
@@ -102,37 +79,8 @@ public class MixedVersionInJvmSut extends InJvmSutBase<IUpgradeableInstance, Upg
             }
         }
 
-        protected InJvmSutBase<IUpgradeableInstance, UpgradeableCluster> sut(UpgradeableCluster cluster)
+        protected InJvmSutBase<IUpgradeableInstance, ICluster<IUpgradeableInstance>> sut(ICluster<IUpgradeableInstance> cluster)
         {
             return new MixedVersionInJvmSut(cluster, initialVersion, upgradeVersions);
         }
     }
-
-    @Override
-    public void afterSchemaInit()
-    {
-        for (int i = 1; i <= cluster.size(); i++)
-        {
-            Versions.Version v = versions.get(i - 1);
-            if (!v.equals(initialVersion))
-            {
-                logger.info("Upgrading {} node from {} to {}", i, initialVersion, v);
-                IUpgradeableInstance instance = cluster.get(i);
-                try
-                {
-                    instance.shutdown().get();
-                }
-                catch (Throwable e)
-                {
-                    throw new RuntimeException(e);
-                }
-                instance.setVersion(v);
-                instance.startup();
-            }
-            else
-            {
-                logger.info("Skipping {} node upgrade, since it is already at the required version ({})", i, initialVersion);
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/harry-integration/src/harry/runner/Reproduce.java b/harry-integration/src/harry/runner/Reproduce.java
deleted file mode 100644
index 5d34ecd..0000000
--- a/harry-integration/src/harry/runner/Reproduce.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- *  Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package harry.runner;
-
-import java.io.File;
-
-import harry.core.Configuration;
-import harry.core.Run;
-import harry.model.sut.PrintlnSut;
-import harry.operations.Query;
-import harry.reconciler.Reconciler;
-
-public class Reproduce
-{
-
-    public static void main(String[] args) throws Throwable
-    {
-        File configFile = HarryRunner.loadConfig(args);
-        Configuration configuration = Configuration.fromFile(configFile);
-        System.out.println(Configuration.toYamlString(configuration));
-        configuration = configuration.unbuild().setSUT(PrintlnSut::new).build();
-
-        Run run = configuration.createRun();
-
-        Reconciler reconciler = new Reconciler(run);
-        long pd = 8135884698435133227L;
-        System.out.println(reconciler.inflatePartitionState(pd,
-                                                            5908L,
-                                                            Query.selectPartition(run.schemaSpec,
-                                                                                  pd,
-                                                                                  false))
-                                     .toString(run.schemaSpec));
-
-        // Try out everything you might want to try with a given run
-    }
-}
diff --git a/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java b/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
index 0da7743..6c4d1d0 100644
--- a/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
+++ b/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
@@ -30,6 +30,9 @@ public class DataGeneratorsIntegrationTest extends CQLTester
     public void testTimestampTieResolution() throws Throwable
     {
         Random rng = new Random(1);
+        String ks = "test_timestamp_tie_resolution";
+        createKeyspace(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}", ks));
+        int counter = 0;
         for (ColumnSpec.DataType<?> dataType : new ColumnSpec.DataType[]{ ColumnSpec.int8Type,
                                                                        ColumnSpec.int16Type,
                                                                        ColumnSpec.int32Type,
@@ -38,7 +41,11 @@ public class DataGeneratorsIntegrationTest extends CQLTester
                                                                        ColumnSpec.floatType,
                                                                        ColumnSpec.doubleType })
         {
-            createTable(String.format("CREATE TABLE %%s (pk int PRIMARY KEY, v %s)",
+
+
+            String tbl = "table_" + (counter++);
+            createTable(String.format("CREATE TABLE %s.%s (pk int PRIMARY KEY, v %s)",
+                                      ks, tbl,
                                       dataType.toString()));
             for (int i = 0; i < 10_000; i++)
             {
@@ -46,15 +53,15 @@ public class DataGeneratorsIntegrationTest extends CQLTester
                 long d2 = dataType.generator().adjustEntropyDomain(rng.nextLong());
                 for (long d : new long[]{ d1, d2 })
                 {
-                    execute("INSERT INTO %s (pk, v) VALUES (?,?) USING TIMESTAMP 1",
+                    execute(String.format("INSERT INTO %s.%s (pk, v) VALUES (?,?) USING TIMESTAMP 1", ks,tbl),
                             i, dataType.generator().inflate(d));
                 }
 
                 if (dataType.compareLexicographically(d1, d2) > 0)
-                    assertRows(execute("SELECT v FROM %s WHERE pk=?", i),
+                    assertRows(execute(String.format("SELECT v FROM %s.%s WHERE pk=?", ks, tbl), i),
                                row(dataType.generator().inflate(d1)));
                 else
-                    assertRows(execute("SELECT v FROM %s WHERE pk=?", i),
+                    assertRows(execute(String.format("SELECT v FROM %s.%s WHERE pk=?", ks, tbl), i),
                                row(dataType.generator().inflate(d2)));
             }
         }
diff --git a/harry-integration/test/harry/model/ConcurrentQuiescentCheckerIntegrationTest.java b/harry-integration/test/harry/model/ConcurrentQuiescentCheckerIntegrationTest.java
new file mode 100644
index 0000000..e0fc67d
--- /dev/null
+++ b/harry-integration/test/harry/model/ConcurrentQuiescentCheckerIntegrationTest.java
@@ -0,0 +1,70 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package harry.model;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+import org.junit.Test;
+
+import harry.core.Configuration;
+import harry.ddl.SchemaGenerators;
+import harry.ddl.SchemaSpec;
+import harry.runner.LockingDataTracker;
+import harry.runner.Runner;
+import harry.visitors.MutatingVisitor;
+import harry.visitors.RandomPartitionValidator;
+
+import static harry.core.Configuration.VisitorPoolConfiguration.pool;
+import static java.util.Arrays.asList;
+
+public class ConcurrentQuiescentCheckerIntegrationTest extends ModelTestBase
+{
+    @Test
+    public void testConcurrentReadWriteWorkload() throws Throwable
+    {
+        Supplier<SchemaSpec> supplier = SchemaGenerators.progression(1);
+        int writeThreads = 2;
+        int readThreads = 2;
+
+        for (int i = 0; i < SchemaGenerators.GENERATORS_COUNT; i++)
+        {
+            SchemaSpec schema = supplier.get();
+
+            Configuration config = configuration(i, schema)
+                                   .setKeyspaceDdl("CREATE KEYSPACE IF NOT EXISTS " + schema.keyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};")
+                                   .setCreateSchema(true)
+                                   .setDropSchema(true)
+                                   .setDataTracker(LockingDataTracker::new)
+                                   .build();
+
+            Runner.concurrent(config,
+                              asList(pool("Writer", writeThreads, MutatingVisitor::new),
+                                     pool("Reader", readThreads, RandomPartitionValidator::new)),
+                              2, TimeUnit.MINUTES)
+                  .run();
+        }
+    }
+
+    @Override
+    Configuration.ModelConfiguration modelConfiguration()
+    {
+        return new Configuration.QuiescentCheckerConfig();
+    }
+}
\ No newline at end of file
diff --git a/harry-integration/test/harry/model/InJVMTokenAwareExecutorTest.java b/harry-integration/test/harry/model/InJVMTokenAwareExecutorTest.java
index c84c789..0ed596b 100644
--- a/harry-integration/test/harry/model/InJVMTokenAwareExecutorTest.java
+++ b/harry-integration/test/harry/model/InJVMTokenAwareExecutorTest.java
@@ -18,6 +18,7 @@
 
 package harry.model;
 
+import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 
 import org.junit.Before;
@@ -28,12 +29,12 @@ import harry.core.Configuration;
 import harry.core.Run;
 import harry.ddl.SchemaGenerators;
 import harry.ddl.SchemaSpec;
-import harry.model.sut.InJVMTokenAwareVisitExecutor;
-import harry.model.sut.InJvmSut;
 import harry.model.sut.SystemUnderTest;
-import harry.runner.RepairingLocalStateValidator;
+import harry.model.sut.injvm.InJVMTokenAwareVisitExecutor;
+import harry.model.sut.injvm.InJvmSut;
+import harry.runner.Runner;
+import harry.runner.UpToLtsRunner;
 import harry.visitors.MutatingVisitor;
-import harry.visitors.Visitor;
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.api.Feature;
 
@@ -58,7 +59,7 @@ public class InJVMTokenAwareExecutorTest extends IntegrationTestBase
     }
 
     @Test
-    public void testRepair()
+    public void testRepair() throws Throwable
     {
         Supplier<SchemaSpec> schemaGen = SchemaGenerators.progression(1);
         for (int cnt = 0; cnt < SchemaGenerators.DEFAULT_RUNS; cnt++)
@@ -70,21 +71,14 @@ public class InJVMTokenAwareExecutorTest extends IntegrationTestBase
             Run run = configuration.createRun();
             run.sut.schemaChange(run.schemaSpec.compile().cql());
 
-            Visitor visitor = new MutatingVisitor(run, new InJVMTokenAwareVisitExecutor(run,
-                                                                                        new Configuration.MutatingRowVisitorConfiguration(),
-                                                                                        SystemUnderTest.ConsistencyLevel.NODE_LOCAL));
 
-            OpSelectors.MonotonicClock clock = run.clock;
-            long maxPd = 0;
-            for (int i = 0; i < 10000; i++)
-            {
-                visitor.visit();
-                maxPd = Math.max(maxPd, run.pdSelector.positionFor(clock.peek()));
-            }
-
-            RepairingLocalStateValidator validator = new RepairingLocalStateValidator(5, 1, run, new Configuration.QuiescentCheckerConfig());
-            validator.visit();
+            Runner.chain(configuration,
+                         UpToLtsRunner.factory(MutatingVisitor.factory(InJVMTokenAwareVisitExecutor.factory(new Configuration.MutatingRowVisitorConfiguration(),
+                                                                                                            SystemUnderTest.ConsistencyLevel.NODE_LOCAL,
+                                                                                                            3)),
+                                               10_000, 2, TimeUnit.SECONDS),
+                         Runner.single(RepairingLocalStateValidator.factory(5, (v) -> true, QuiescentChecker::new)))
+                  .run();
         }
-
     }
 }
diff --git a/harry-integration/test/harry/model/IntegrationTestBase.java b/harry-integration/test/harry/model/IntegrationTestBase.java
index 6d8b4ef..9eb8a7e 100644
--- a/harry-integration/test/harry/model/IntegrationTestBase.java
+++ b/harry-integration/test/harry/model/IntegrationTestBase.java
@@ -28,8 +28,10 @@ import harry.core.Configuration;
 import harry.ddl.SchemaGenerators;
 import harry.ddl.SchemaSpec;
 import harry.model.clock.OffsetClock;
-import harry.model.sut.InJvmSut;
+import harry.model.sut.injvm.InJvmSut;
+import harry.model.sut.injvm.InJvmSutBase;
 import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
 
 public class IntegrationTestBase extends TestBaseImpl
 {
@@ -41,8 +43,9 @@ public class IntegrationTestBase extends TestBaseImpl
     {
         cluster = init(Cluster.build()
                               .withNodes(1)
+                              .withConfig(InJvmSutBase.defaultConfig())
                               .start());
-        sut = new InJvmSut(cluster, 1);
+        sut = new InJvmSut(cluster, 20);
     }
 
     @AfterClass
@@ -97,7 +100,7 @@ public class IntegrationTestBase extends TestBaseImpl
                                                        .setDropSchema(true)
                                                        .setSchemaProvider((seed1, sut) -> schema)
                                                        .setClusteringDescriptorSelector(sharedCDSelectorConfiguration().build())
-                                                       .setPartitionDescriptorSelector(new Configuration.DefaultPDSelectorConfiguration(1, 200))
+                                                       .setPartitionDescriptorSelector(new Configuration.DefaultPDSelectorConfiguration(2, 200))
                                                        .setSUT(() -> sut);
     }
 }
\ No newline at end of file
diff --git a/harry-integration/test/harry/model/ModelTestBase.java b/harry-integration/test/harry/model/ModelTestBase.java
index 3381137..0baee11 100644
--- a/harry-integration/test/harry/model/ModelTestBase.java
+++ b/harry-integration/test/harry/model/ModelTestBase.java
@@ -18,6 +18,8 @@
 
 package harry.model;
 
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.concurrent.TimeUnit;
 import java.util.function.BiConsumer;
 import java.util.function.Function;
@@ -27,15 +29,17 @@ import harry.core.Configuration;
 import harry.core.Run;
 import harry.ddl.SchemaGenerators;
 import harry.ddl.SchemaSpec;
-import harry.visitors.LoggingVisitor;
-import harry.visitors.MutatingRowVisitor;
 import harry.runner.Runner;
+import harry.runner.UpToLtsRunner;
+import harry.visitors.MutatingRowVisitor;
+import harry.visitors.MutatingVisitor;
 import harry.visitors.SingleValidator;
-import harry.visitors.Visitor;
 
 public abstract class ModelTestBase extends IntegrationTestBase
 {
-    void negativeTest(Function<Run, Boolean> corrupt, BiConsumer<Throwable, Run> validate)
+    private final int ITERATIONS = 20_000;
+
+    void negativeTest(Function<Run, Boolean> corrupt, BiConsumer<Throwable, Run> validate) throws Throwable
     {
         Supplier<SchemaSpec> supplier = SchemaGenerators.progression(SchemaGenerators.DEFAULT_SWITCH_AFTER);
         for (int i = 0; i < SchemaGenerators.DEFAULT_RUNS; i++)
@@ -62,18 +66,10 @@ public abstract class ModelTestBase extends IntegrationTestBase
             Configuration config = builder.build();
             Runner runner = config.createRunner();
             
-            try
-            {
-                Run run = runner.getRun();
-                beforeEach();
-                run.sut.schemaChange(run.schemaSpec.compile().cql());
-
-                runner.initAndStartAll().get(4, TimeUnit.MINUTES);
-            }
-            finally
-            {
-                runner.shutdown();
-            }
+            Run run = runner.getRun();
+            beforeEach();
+            run.sut.schemaChange(run.schemaSpec.compile().cql());
+            runner.run();
         }
     }
 
@@ -89,37 +85,54 @@ public abstract class ModelTestBase extends IntegrationTestBase
         return sharedConfiguration(seed, schema);
     }
 
-    void negativeTest(Function<Run, Boolean> corrupt, BiConsumer<Throwable, Run> validate, int counter, SchemaSpec schemaSpec)
+    void negativeTest(Function<Run, Boolean> corrupt, BiConsumer<Throwable, Run> validate, int counter, SchemaSpec schemaSpec) throws Throwable
     {
-        Configuration config = configuration(counter, schemaSpec).build();
+        Configuration config = configuration(counter, schemaSpec)
+                               .setCreateSchema(true)
+                               .setKeyspaceDdl(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': %d};",
+                                                             schemaSpec.keyspace, cluster.size()))
+                               .setDropSchema(true)
+                               .build();
 
         Run run = config.createRun();
-        beforeEach();
-        run.sut.schemaChange(run.schemaSpec.compile().cql());
-        System.out.println(run.schemaSpec.compile().cql());
-
-        Visitor visitor = new LoggingVisitor(run, MutatingRowVisitor::new);
 
-        for (int i = 0; i < 20000; i++)
-            visitor.visit();
+        new Runner.ChainRunner(run, config,
+                               Arrays.asList(writer(ITERATIONS, 2, TimeUnit.MINUTES),
+                                             (r,  c) -> new Runner.SingleVisitRunner(r, c, Collections.singletonList(this::validator)) {
+                                                 @Override
+                                                 public void runInternal()
+                                                 {
+                                                     if (!corrupt.apply(run))
+                                                     {
+                                                         System.out.println("Could not corrupt");
+                                                         return;
+                                                     }
+                                                     try
+                                                     {
+                                                         super.runInternal();
+                                                         throw new ShouldHaveThrownException();
+                                                     }
+                                                     catch (Throwable t)
+                                                     {
+                                                         validate.accept(t, run);
+                                                     }
+                                                 }
+                                             })).run();
+    }
 
-        SingleValidator validator = validator(run);
-        validator.visit(0);
+    public static Configuration.RunnerConfiguration writer(long iterations, int runtime, TimeUnit timeUnit)
+    {
+        return (run, config) -> {
+            return new UpToLtsRunner(run, config,
+                                     Collections.singletonList((r_) -> new MutatingVisitor(r_, MutatingRowVisitor::new)),
+                                     iterations,
+                                     runtime, timeUnit);
+        };
+    }
 
-        if (!corrupt.apply(run))
-        {
-            System.out.println("Could not corrupt");
-            return;
-        }
+    public static class ShouldHaveThrownException extends AssertionError
+    {
 
-        try
-        {
-            validator.visit(0);
-        }
-        catch (Throwable t)
-        {
-            validate.accept(t, run);
-        }
     }
 }
 
diff --git a/harry-integration/test/harry/model/QuerySelectorNegativeTest.java b/harry-integration/test/harry/model/QuerySelectorNegativeTest.java
index 13ae548..95bcbc8 100644
--- a/harry-integration/test/harry/model/QuerySelectorNegativeTest.java
+++ b/harry-integration/test/harry/model/QuerySelectorNegativeTest.java
@@ -78,6 +78,7 @@ public class QuerySelectorNegativeTest extends IntegrationTestBase
                                                                        HideRowCorruptor::new),
                              (run) -> new AddExtraRowCorruptor(run.schemaSpec,
                                                                run.clock,
+                                                               run.tracker,
                                                                run.descriptorSelector)
         );
     }
diff --git a/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java b/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
index 97a554a..9ce991b 100644
--- a/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
+++ b/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
@@ -45,18 +45,25 @@ import harry.visitors.SingleValidator;
 
 public class QuiescentCheckerIntegrationTest extends ModelTestBase
 {
+    private final long CORRUPT_LTS = 0L;
+
     @Override
     protected SingleValidator validator(Run run)
     {
-        return new SingleValidator(100, run, modelConfiguration());
+        return new SingleValidator(100, run, modelConfiguration()) {
+            public void visit()
+            {
+                visit(CORRUPT_LTS);
+            }
+        };
     }
 
     @Test
-    public void testNormalCondition()
+    public void testNormalCondition() throws Throwable
     {
         negativeTest((run) -> true,
                      (t, run) -> {
-                         if (t != null)
+                         if (!(t instanceof ShouldHaveThrownException))
                              throw new AssertionError(String.format("Throwable was supposed to be null. Schema: %s",
                                                                     run.schemaSpec.compile().cql()),
                                                       t);
@@ -82,7 +89,7 @@ public class QuiescentCheckerIntegrationTest extends ModelTestBase
         Model.ModelFactory factory = modelConfiguration();
 
         ConcurrentRunnerConfig concurrent = 
-                new ConcurrentRunnerConfig(4, Collections.singletonList(new LoggingVisitorConfiguration(new Configuration.MutatingRowVisitorConfiguration())),
+                new ConcurrentRunnerConfig(Arrays.asList(new Configuration.VisitorPoolConfiguration("Writer", 4, new Configuration.MutatingVisitorConfiguation(new Configuration.MutatingRowVisitorConfiguration()))),
                                            30, TimeUnit.SECONDS);
         SingleVisitRunnerConfig sequential = 
                 new SingleVisitRunnerConfig(Collections.singletonList(new RecentPartitionsValidatorConfiguration(1024, 0, 1, factory::make)));
@@ -93,7 +100,7 @@ public class QuiescentCheckerIntegrationTest extends ModelTestBase
     }
 
     @Test
-    public void testDetectsMissingRow()
+    public void testDetectsMissingRow() throws Throwable
     {
         negativeTest((run) -> {
                          SimpleQueryResponseCorruptor corruptor = new SimpleQueryResponseCorruptor(run.schemaSpec,
@@ -101,7 +108,7 @@ public class QuiescentCheckerIntegrationTest extends ModelTestBase
                                                                                                    HideRowCorruptor::new);
 
                          Query query = Query.selectPartition(run.schemaSpec,
-                                                             run.pdSelector.pd(0, run.schemaSpec),
+                                                             run.pdSelector.pd(CORRUPT_LTS, run.schemaSpec),
                                                              false);
 
                          return corruptor.maybeCorrupt(query, run.sut);
@@ -120,23 +127,24 @@ public class QuiescentCheckerIntegrationTest extends ModelTestBase
     }
 
     @Test
-    public void testDetectsExtraRow()
+    public void testDetectsExtraRow() throws Throwable
     {
         negativeTest((run) -> {
                          QueryResponseCorruptor corruptor = new AddExtraRowCorruptor(run.schemaSpec,
                                                                                      run.clock,
+                                                                                     run.tracker,
                                                                                      run.descriptorSelector);
 
                          return corruptor.maybeCorrupt(Query.selectPartition(run.schemaSpec,
-                                                                             run.pdSelector.pd(0, run.schemaSpec),
+                                                                             run.pdSelector.pd(CORRUPT_LTS, run.schemaSpec),
                                                                              false),
                                                        run.sut);
                      },
                      (t, run) -> {
                          String expected = "Found a row in the model that is not present in the resultset";
                          String expected2 = "Expected results to have the same number of results, but actual result iterator has more results";
-
-                         if (t.getMessage().contains(expected) || t.getMessage().contains(expected2))
+                         String expected3 = "Found a row while model predicts statics only";
+                         if (t.getMessage().contains(expected) || t.getMessage().contains(expected2) || t.getMessage().contains(expected3))
                              return;
 
                          throw new AssertionError(String.format("Exception string mismatch.\nExpected error: %s.\nActual error: %s", expected, t.getMessage()),
@@ -146,7 +154,7 @@ public class QuiescentCheckerIntegrationTest extends ModelTestBase
 
 
     @Test
-    public void testDetectsRemovedColumn()
+    public void testDetectsRemovedColumn() throws Throwable
     {
         negativeTest((run) -> {
                          SimpleQueryResponseCorruptor corruptor = new SimpleQueryResponseCorruptor(run.schemaSpec,
@@ -154,7 +162,7 @@ public class QuiescentCheckerIntegrationTest extends ModelTestBase
                                                                                                    HideValueCorruptor::new);
 
                          return corruptor.maybeCorrupt(Query.selectPartition(run.schemaSpec,
-                                                                             run.pdSelector.pd(0, run.schemaSpec),
+                                                                             run.pdSelector.pd(CORRUPT_LTS, run.schemaSpec),
                                                                              false),
                                                        run.sut);
                      },
@@ -172,7 +180,7 @@ public class QuiescentCheckerIntegrationTest extends ModelTestBase
 
 
     @Test
-    public void testDetectsOverwrittenRow()
+    public void testDetectsOverwrittenRow() throws Throwable
     {
         negativeTest((run) -> {
                          SimpleQueryResponseCorruptor corruptor = new SimpleQueryResponseCorruptor(run.schemaSpec,
@@ -180,7 +188,7 @@ public class QuiescentCheckerIntegrationTest extends ModelTestBase
                                                                                                    ChangeValueCorruptor::new);
 
                          return corruptor.maybeCorrupt(Query.selectPartition(run.schemaSpec,
-                                                                             run.pdSelector.pd(0, run.schemaSpec),
+                                                                             run.pdSelector.pd(CORRUPT_LTS, run.schemaSpec),
                                                                              false),
                                                        run.sut);
                      },
diff --git a/harry-integration/test/harry/model/QuiescentLocalStateCheckerIntegrationTest.java b/harry-integration/test/harry/model/QuiescentLocalStateCheckerIntegrationTest.java
new file mode 100644
index 0000000..ae537db
--- /dev/null
+++ b/harry-integration/test/harry/model/QuiescentLocalStateCheckerIntegrationTest.java
@@ -0,0 +1,79 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package harry.model;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import harry.core.Configuration;
+import harry.ddl.SchemaGenerators;
+import harry.ddl.SchemaSpec;
+import harry.model.sut.injvm.InJvmSut;
+import harry.model.sut.injvm.InJvmSutBase;
+import harry.model.sut.injvm.QuiescentLocalStateChecker;
+import harry.runner.Runner;
+import harry.visitors.AllPartitionsValidator;
+import harry.visitors.MutatingVisitor;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.Feature;
+
+public class QuiescentLocalStateCheckerIntegrationTest extends ModelTestBase
+{
+    @BeforeClass
+    public static void before() throws Throwable
+    {
+        cluster = init(Cluster.build()
+                              .withNodes(2)
+                              .withConfig(InJvmSutBase.defaultConfig().andThen((cfg) -> cfg.with(Feature.GOSSIP)))
+                              .start());
+        sut = new InJvmSut(cluster, 1);
+    }
+
+
+    @Test
+    public void testQuiescentLocalStateChecker() throws Throwable
+    {
+        Supplier<SchemaSpec> supplier = SchemaGenerators.progression(1);
+        for (int i = 0; i < SchemaGenerators.GENERATORS_COUNT; i++)
+        {
+            SchemaSpec schema = supplier.get();
+
+            Configuration config = configuration(i, schema)
+                                   .setKeyspaceDdl("CREATE KEYSPACE IF NOT EXISTS " + schema.keyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};")
+                                   .setCreateSchema(true)
+                                   .setDropSchema(true)
+                                   .build();
+
+            Runner.chain(config,
+                         Runner.sequential(MutatingVisitor::new, 2, TimeUnit.SECONDS),
+                         Runner.single(AllPartitionsValidator.factory(1, (p) -> true, QuiescentLocalStateChecker.factory(1))))
+                  .run();
+            break;
+        }
+    }
+
+    @Override
+    Configuration.ModelConfiguration modelConfiguration()
+    {
+        return new Configuration.QuiescentCheckerConfig();
+    }
+}
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index c044783..c749d67 100755
--- a/pom.xml
+++ b/pom.xml
@@ -156,7 +156,7 @@
             <dependency>
                 <groupId>org.reflections</groupId>
                 <artifactId>reflections</artifactId>
-                <version>0.9.12</version>
+                <version>0.10.2</version>
             </dependency>
 
 	    <dependency>


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org