You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by if...@apache.org on 2023/03/13 10:01:22 UTC
[cassandra-harry] 04/06: Implement concurrent quiescent checker
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch CASSANDRA-18318
in repository https://gitbox.apache.org/repos/asf/cassandra-harry.git
commit e465026ba243deb21e79b94de481973c53aad3b7
Author: Alex Petrov <al...@apple.com>
AuthorDate: Fri Mar 3 09:38:59 2023 +0100
Implement concurrent quiescent checker
Patch by Alex Petrov; reviewed by Caleb Rackliffe and Abe Ratnofsky for CASSANDRA-18315
---
Makefile | 18 +-
README.md | 104 ++++-
bin/build-shaded-dtest-for-harry.sh | 51 +++
bin/relocate-dependencies-harry.pom | 154 ++++++++
conf/default.yaml | 3 -
conf/external.yaml | 41 +-
harry-core/pom.xml | 10 +
harry-core/src/harry/core/Configuration.java | 178 ++++++---
harry-core/src/harry/core/Run.java | 2 -
.../src/harry/corruptor/AddExtraRowCorruptor.java | 11 +-
harry-core/src/harry/data/ResultSetRow.java | 30 +-
harry-core/src/harry/ddl/SchemaGenerators.java | 5 +-
harry-core/src/harry/dsl/HistoryBuilder.java | 5 +
.../harry/model/AlwaysSamePartitionSelector.java | 5 +
harry-core/src/harry/model/OpSelectors.java | 26 +-
.../src/harry/model/QueryingNoOpValidator.java | 21 +-
harry-core/src/harry/model/QuiescentChecker.java | 41 +-
.../model/QuiescentLocalStateCheckerBase.java | 110 ++++++
.../harry/model}/RepairingLocalStateValidator.java | 95 ++---
harry-core/src/harry/model/clock/OffsetClock.java | 17 +-
.../src/harry/model/sut/TokenPlacementModel.java | 218 +++++++++++
.../sut/injvm}/InJVMTokenAwareVisitExecutor.java | 85 +++-
.../src/harry/model/sut/injvm/InJvmSut.java | 30 +-
.../src/harry/model/sut/injvm}/InJvmSutBase.java | 74 ++--
.../model/sut/injvm/MixedVersionInJvmSut.java | 74 ++++
.../sut/injvm/QuiescentLocalStateChecker.java | 76 ++++
harry-core/src/harry/reconciler/Reconciler.java | 62 +--
harry-core/src/harry/runner/DataTracker.java | 69 ++--
.../src/harry/runner/DefaultDataTracker.java | 146 ++++---
harry-core/src/harry/runner/HarryRunner.java | 64 +---
.../src/harry/runner/LockingDataTracker.java | 257 +++++++++++++
harry-core/src/harry/runner/Runner.java | 426 ++++++++-------------
harry-core/src/harry/runner/StagedRunner.java | 133 +------
.../src/harry/runner/TrivialShrinker.java | 4 +-
harry-core/src/harry/runner/UpToLtsRunner.java | 46 ++-
harry-core/src/harry/util/ByteUtils.java | 47 +--
.../src/harry/util/StringUtils.java | 23 +-
.../src/harry/visitors/AllPartitionsValidator.java | 146 +++----
.../src/harry/visitors/CorruptingVisitor.java | 1 +
.../src/harry/visitors}/FaultInjectingVisitor.java | 48 ++-
harry-core/src/harry/visitors/MutatingVisitor.java | 40 +-
.../harry/visitors/RandomPartitionValidator.java | 61 +++
.../src/harry/visitors/SkippingVisitor.java | 0
harry-core/test/harry/operations/RelationTest.java | 5 +
.../test/harry/runner/LockingDataTrackerTest.java | 120 ++++++
.../dependency-reduced-pom.xml | 2 +-
harry-integration-external/pom.xml | 2 +-
.../com/datastax/driver/core/UtilsAccessor.java | 15 +-
.../model/sut/external/ExternalClusterSut.java | 56 ++-
.../sut/external/QuiescentLocalStateChecker.java | 80 ++++
.../harry/runner/external/HarryRunnerExternal.java | 9 +-
.../src/harry/runner/external/MiniStress.java | 126 ++++++
harry-integration/pom.xml | 4 +
.../src/harry/model/sut/ByteUtils.java | 115 ------
.../InJvmSutConfiguration.java} | 53 +--
.../MixedInJvmSutConfiguration.java} | 66 +---
harry-integration/src/harry/runner/Reproduce.java | 52 ---
.../generators/DataGeneratorsIntegrationTest.java | 15 +-
.../ConcurrentQuiescentCheckerIntegrationTest.java | 70 ++++
.../harry/model/InJVMTokenAwareExecutorTest.java | 32 +-
.../test/harry/model/IntegrationTestBase.java | 9 +-
.../test/harry/model/ModelTestBase.java | 93 +++--
.../harry/model/QuerySelectorNegativeTest.java | 1 +
.../model/QuiescentCheckerIntegrationTest.java | 36 +-
.../QuiescentLocalStateCheckerIntegrationTest.java | 79 ++++
pom.xml | 2 +-
66 files changed, 2697 insertions(+), 1402 deletions(-)
diff --git a/Makefile b/Makefile
index c43804c..31201df 100644
--- a/Makefile
+++ b/Makefile
@@ -14,13 +14,24 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+CASSANDRA_SHA = 39ded1844f
DOCKER_REPO = org.apache.cassandra/harry/harry-runner
-mvn:
+cassandra:
+ git clone https://github.com/apache/cassandra.git
+
+cassandra-jar: cassandra
+ cp bin/build-shaded-dtest-for-harry.sh ./cassandra/
+ cp bin/relocate-dependencies-harry.pom ./cassandra/
+ cd cassandra; \
+ git checkout ${CASSANDRA_SHA}; \
+ ./build-shaded-dtest-for-harry.sh
+
+package: cassandra-jar
rm -fr shared/*
mvn clean && mvn package -DskipTests
-img: mvn
+img: package
docker build -t ${DOCKER_REPO}:latest-local ./ -f docker/Dockerfile.local
run: img
@@ -32,3 +43,6 @@ run-last:
standalone:
rm -fr shared/*
mvn clean && mvn package -DskipTests -P standalone
+
+stress: package
+ @java -cp $(shell pwd)/harry-core/target/lib/*:$(shell pwd)/harry-integration/target/lib/*:$(shell pwd)/harry-integration-external/target/lib/*:$(shell pwd)/harry-core/target/*:$(shell pwd)/harry-integration/target/*:$(shell pwd)/harry-integration-external/target/* -ea harry.runner.external.MiniStress $(ARGS)
diff --git a/README.md b/README.md
index 9c6b7bf..a09a345 100644
--- a/README.md
+++ b/README.md
@@ -4,6 +4,94 @@ The project aims to generate _reproducible_ workloads that are as close to real-
as possible, while being able to _efficiently_ verify the cluster state against
the model without pausing the workload itself.
+# TL;DR
+
+One way to see Harry is as a read/write stress tool that can check if the read results
+are consistent with what it knows that it has written. This means that in order to test
+Apache Cassandra with Harry, all you need is to run Harry.
+
+If you're using an external cluster (i.e. `./bin/cassandra -f`, CCM, docker, kubernetes,
+or just a deployed cluster), just use the command line mini stress tool:
+
+```
+make stress ARGS="conf/external.yaml --read=2 --write=2 --duration=60000 --validate-all-local"
+```
+
+This command will start perform concurrent read/write workload, 2 read and 2 write threads for
+60 seconds, and will validate presence of each written row on each node locally when it
+concludes. If it sees any inconsistencies, it will inform you accordingly. There are plenty
+configurables in the configuration files (schema, partition size, timestamp generation rules,
+authentication params, ratios for different mutation query types, and many more).
+
+An equivalent for an in-jvm cluster is:
+
+```
+try (Cluster cluster = builder().withNodes(3)
+ .start())
+{
+ SchemaSpec schema = new SchemaSpec("harry", "test_table",
+ asList(pk("pk1", asciiType), pk("pk1", int64Type)),
+ asList(ck("ck1", asciiType), ck("ck1", int64Type)),
+ asList(regularColumn("regular1", asciiType), regularColumn("regular1", int64Type)),
+ asList(staticColumn("static1", asciiType), staticColumn("static1", int64Type)));
+
+ Configuration config = Configuration.fromFile("conf/example.yaml")
+ .unbuild()
+ .setKeyspaceDdl(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': %d};", schema.keyspace, 3))
+ .setSUT(() -> new InJvmSut(cluster))
+ .build();
+
+ Run run = config.createRun();
+
+ concurrent(run, config,
+ asList(pool("Writer", 2, MutatingVisitor::new),
+ pool("Reader", 2, RandomPartitionValidator::new)),
+ 2, TimeUnit.MINUTES)
+ .run();
+}
+```
+
+# What's available
+
+## System Under Test implementations
+
+ * `in_jvm/InJvmSut` - simple in-JVM-dtest system under test.
+ * `println/PrintlnSut` - system under test that prints to sdtout instead of executing queries on the cluster; useful for debugging.
+ * `mixed_in_jvm/MixedVersionInJvmSut` - in-JVM-dtest system under test that works with mixed version clusters.
+ * `external/ExternalClusterSut` - system under test that works with CCM, Docker, Kubernetes, or cluster you may.
+ have deployed elsewhere
+
+Both in-JVM SUTs have fault-injecting functionality available.
+
+## Visitors
+
+ * `single/SingleValidator` - visitor that runs several different read queries against a single partition that is associated with current logical timestamp, and validates their results using given model.
+ * `all_partitions/AllPartitionsValidator` - concurrently validates all partitions that were visited during this run.
+ * `repair_and_validate_local_states/RepairingLocalStateValidator` - similar to `AllPartitionsValidator`, but performs repair before checking node states.
+ * `mutating/MutatingVisitor` - visitor that performs all sorts of mutations.
+ * `logging/LoggingVisitor` - similar to `MutatingVisitor`, but also logs all operations to a file; useful for debug purposes.
+ * `corrupting/CorruptingVisitor` - visitor that will deliberately change data in the partition it visits. Useful for negative tests (i.e. to ensure that your model actually detects data inconsistencies).
+
+And more.
+
+## Models
+
+ * `querying_no_op/QueryingNoOpValidator` - a model that can be used to "simply" run random queries.
+ * `quiescent_checker/QuiescentChecker` - a model that can be used to verify results of any read that has no writes to the same partition_ concurrent to it. Should be used in conjunction with locking data tracker.
+ * `quiescent_local_state_checker/QuiescentLocalStateChecker` - a model that can check local states of each replica that has to own
+
+## Runners
+
+ * `sequential/SequentialRunner` - runs all visitors sequentially, in the loop, for a specified amount of time; useful for simple tests that do not have to exercise concurrent read/write path.
+ * `concurrent/ConcurrentRunner` - runs all visitors concurrently, each visitor in its own thread, looped, for a specified amount of time; useful for things like concurrent read/write workloads.
+ * `chain/ChainRunner` - receives other runners as input, and runs them one after another once. Useful for both simple and complex scenarios that involve both read/write workloads, validating all partitions, exercising other node-local or cluster-wide operations.
+ * `staged/StagedRunner` - receives other runners (stages) as input, and runs them one after another in a loop; useful for implementing complex scenarios, such as read/write workloads followed by some cluster changing operations.
+
+## Clock
+
+ * `approximate_monotonic/ApproxomateMonotonicClock` - a timestamp supplier implementation that tries to keep as close to real time as possible, while preserving mapping from real-time to logical timestamps.
+ * `offset/OffsetClock` - a (monotonic) clock that supplies timestamps that do not have any relation to real time.
+
# Introduction
Harry has two primary modes of functionality:
@@ -653,17 +741,13 @@ of writing, there's no official repository where these jars are released, so you
have to build it manually:
```
-git clone git@github.com:apache/cassandra.git
-cd cassandra
-./build-shaded-dtest-jar.sh 4.0-beta4 4.0.0-SNAPSHOT
-cd ~/../harry/
-mvn package -DskipTests -nsu
+make package
mvn dependency:copy-dependencies
-java -cp harry-runner/target/harry-runner-0.0.1-SNAPSHOT.jar:$(find harrry-runner/target/dependency/*.jar | tr -s '\n' ':'). harry.runner.HarryRunner
-```
+java -cp harry-core/target/harry-core-0.0.2-SNAPSHOT.jar:$(find harrry-core/target/dependency/*.jar | tr -s '\n' ':'). harry.runner.HarryRunner
+
-`4.0-beta3` is a version of Cassandra which you can find in `build.xml`, and
-`4.0.0-SNAPSHOT` is a version of dtest jar that'll be installed under
+`4.2` is a version of Cassandra which you can find in `build.xml`, and
+`4.2-SNAPSHOT` is a version of dtest jar that'll be installed under
`org.apache.cassandra:cassandra-dtest-local` in your `~/.m2/repository`.
Alternatively, you can use a Docker container. For that just run:
@@ -671,7 +755,7 @@ Alternatively, you can use a Docker container. For that just run:
```
git clone git@github.com:apache/cassandra.git
cd cassandra
-./build-shaded-dtest-jar.sh 4.0-beta4 4.0.0-SNAPSHOT
+./build-shaded-dtest-jar.sh 4.2 4.2-SNAPSHOT
cd ~/../harry/
make run
```
diff --git a/bin/build-shaded-dtest-for-harry.sh b/bin/build-shaded-dtest-for-harry.sh
new file mode 100755
index 0000000..e384556
--- /dev/null
+++ b/bin/build-shaded-dtest-for-harry.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+
+export REVISION=`git rev-parse --short HEAD`
+export BASE_VERSION=$(cat build.xml | grep 'property name="base.version"' | awk -F "\"" '{print $4}')
+export VERSION=$BASE_VERSION
+export ARTIFACT_NAME=cassandra-dtest
+export REPO_DIR=~/.m2/repository/
+
+ant clean
+ant dtest-jar
+
+# Install the version that will be shaded
+mvn install:install-file \
+ -Dfile=./build/dtest-${BASE_VERSION}.jar \
+ -DgroupId=org.apache.cassandra \
+ -DartifactId=${ARTIFACT_NAME}-local \
+ -Dversion=${VERSION}-${REVISION} \
+ -Dpackaging=jar \
+ -DgeneratePom=true
+
+# Create shaded artifact
+mvn -f relocate-dependencies-harry.pom versions:set -DnewVersion=${VERSION}-${REVISION} -U
+mvn -f relocate-dependencies-harry.pom package -DskipTests -nsu -Ddtest-local.version=${VERSION}-${REVISION} -U
+
+# Deploy shaded artifact
+mvn install:install-file \
+ -Dfile=./target/${ARTIFACT_NAME}-shaded-${VERSION}-${REVISION}.jar \
+ -DgroupId=org.apache.cassandra \
+ -DartifactId=${ARTIFACT_NAME}-shaded \
+ -Dversion=${VERSION}-${REVISION} \
+ -Dpackaging=jar \
+ -DgeneratePom=true \
+ -DlocalRepositoryPath=${REPO_DIR}
diff --git a/bin/relocate-dependencies-harry.pom b/bin/relocate-dependencies-harry.pom
new file mode 100644
index 0000000..7d0aa3c
--- /dev/null
+++ b/bin/relocate-dependencies-harry.pom
@@ -0,0 +1,154 @@
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.cassandra</groupId>
+ <artifactId>cassandra-dtest-shaded</artifactId>
+ <version>VERSION_PLACEHOLDER</version>
+ <packaging>jar</packaging>
+
+ <name>Cassandra in-jvm dtests shaded jar</name>
+
+ <properties>
+ <project.type>library</project.type>
+ <java.version>1.8</java.version>
+ <test.source.directory>src/test/unit/java</test.source.directory>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+
+ <junit.version>4.12</junit.version>
+ <maven.compiler.source>${java.version}</maven.compiler.source>
+ <maven.compiler.target>${java.version}</maven.compiler.target>
+ <dtest-local.version>DTEST_VERSION_PLACEHOLDER</dtest-local.version>
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.cassandra</groupId>
+ <artifactId>cassandra-dtest-local</artifactId>
+ <version>${dtest-local.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <testSourceDirectory>${test.source.directory}</testSourceDirectory>
+
+ <plugins>
+ <!-- generate a shaded JAR -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.2.4</version>
+
+ <configuration>
+ <createSourcesJar>false</createSourcesJar>
+
+ <artifactSet>
+ <includes>
+ <include>org.apache.cassandra:cassandra-dtest-local</include>
+ </includes>
+ </artifactSet>
+
+ <relocations>
+ <relocation>
+ <pattern>io.netty</pattern>
+ <shadedPattern>relocated.shaded.io.netty</shadedPattern>
+ </relocation>
+
+ <relocation>
+ <pattern>com.google</pattern>
+ <shadedPattern>relocated.shaded.com.google</shadedPattern>
+ </relocation>
+
+ <relocation>
+ <pattern>com.datastax</pattern>
+ <shadedPattern>relocated.shaded.com.datastax</shadedPattern>
+ </relocation>
+ </relocations>
+
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>**/Log4j2Plugins.dat</exclude>
+ <exclude>net/openhft/chronicle/wire/YamlWire$TextValueIn.class</exclude>
+ </excludes>
+ </filter>
+
+ <filter>
+ <artifact>io.netty:netty-*</artifact>
+ <excludes>
+ <exclude>META-INF/maven/</exclude>
+ <exclude>META-INF/io.netty.versions.properties</exclude>
+ </excludes>
+ </filter>
+
+ <filter>
+ <artifact>com.google.guava:guava</artifact>
+ <excludes>
+ <exclude>META-INF/maven/</exclude>
+ </excludes>
+ </filter>
+
+ <filter>
+ <artifact>com.google.guava:failureaccess</artifact>
+ <excludes>
+ <exclude>META-INF/maven/</exclude>
+ </excludes>
+ </filter>
+
+ <filter>
+ <artifact>com.datastax.cassandra:cassandra-driver-core</artifact>
+ <excludes>
+ <exclude>META-INF/maven/</exclude>
+ </excludes>
+ </filter>
+
+ </filters>
+ </configuration>
+
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+
+ </configuration>
+ </execution>
+ </executions>
+
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/conf/default.yaml b/conf/default.yaml
index c8d00b3..a004747 100644
--- a/conf/default.yaml
+++ b/conf/default.yaml
@@ -35,9 +35,6 @@ clock:
epoch_length: 1
epoch_time_unit: "SECONDS"
-run_time: 2
-run_time_unit: "HOURS"
-
# System under test: a Cassandra node or cluster. Default implementation is in_jvm (in-jvm DTest cluster).
# Harry also supports external clusters.
system_under_test:
diff --git a/conf/external.yaml b/conf/external.yaml
index f6be4c7..b75547e 100644
--- a/conf/external.yaml
+++ b/conf/external.yaml
@@ -18,11 +18,25 @@ seed: 1596731732524
# Default schema provider generates random schema
schema_provider:
- default: {}
-
-drop_schema: false
-create_schema: true
-truncate_table: false
+ fixed:
+ keyspace: harry
+ table: test_table
+ partition_keys:
+ pk1: bigint
+ pk2: ascii
+ clustering_keys:
+ ck1: ascii
+ ck2: bigint
+ regular_columns:
+ v1: ascii
+ v2: bigint
+ v3: ascii
+ v4: bigint
+ static_keys:
+ s1: ascii
+ s2: bigint
+ s3: ascii
+ s4: bigint
# Clock is a component responsible for mapping _logical_ timestamps to _real-time_ ones.
#
@@ -35,6 +49,10 @@ clock:
epoch_length: 1
epoch_time_unit: "SECONDS"
+drop_schema: false
+create_schema: true
+truncate_table: true
+
# System under test: a Cassandra node or cluster. Default implementation is in_jvm (in-jvm DTest cluster).
# Harry also supports external clusters.
system_under_test:
@@ -44,12 +62,6 @@ system_under_test:
username: null
password: null
-# Model is responsible for tracking logical timestamps that
-# model:
-# exhaustive_checker:
-# max_seen_lts: 19
-# max_complete_lts: 16
-
# Partition descriptor selector controls how partitions is selected based on the current logical
# timestamp. Default implementation is a sliding window of partition descriptors that will visit
# one partition after the other in the window `slide_after_repeats` times. After that will
@@ -67,7 +79,7 @@ clustering_descriptor_selector:
default:
modifications_per_lts:
type: "constant"
- constant: 4
+ constant: 2
rows_per_modification:
type: "constant"
constant: 2
@@ -114,3 +126,8 @@ runner:
metric_reporter:
no_op: {}
+
+data_tracker:
+ locking:
+ max_seen_lts: -1
+ max_complete_lts: -1
diff --git a/harry-core/pom.xml b/harry-core/pom.xml
index 153251c..a65fe5e 100755
--- a/harry-core/pom.xml
+++ b/harry-core/pom.xml
@@ -32,6 +32,16 @@
<name>Harry Core</name>
<dependencies>
+ <dependency>
+ <groupId>org.apache.cassandra</groupId>
+ <artifactId>dtest-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ </dependency>
+
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
diff --git a/harry-core/src/harry/core/Configuration.java b/harry-core/src/harry/core/Configuration.java
index 7cd8f72..88474b9 100644
--- a/harry-core/src/harry/core/Configuration.java
+++ b/harry-core/src/harry/core/Configuration.java
@@ -18,14 +18,23 @@
package harry.core;
+import java.io.BufferedWriter;
import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.lang.annotation.Annotation;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
+import java.util.stream.Collectors;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -38,14 +47,12 @@ import harry.ddl.SchemaGenerators;
import harry.ddl.SchemaSpec;
import harry.generators.Surjections;
import harry.generators.distribution.Distribution;
-import harry.model.AlwaysSamePartitionSelector;
import harry.model.Model;
import harry.model.OpSelectors;
import harry.model.QuiescentChecker;
import harry.model.clock.ApproximateMonotonicClock;
-import harry.model.clock.OffsetClock;
-import harry.model.sut.PrintlnSut;
import harry.model.sut.SystemUnderTest;
+import harry.runner.LockingDataTracker;
import harry.visitors.AllPartitionsValidator;
import harry.visitors.CorruptingVisitor;
import harry.runner.DataTracker;
@@ -54,55 +61,42 @@ import harry.visitors.LoggingVisitor;
import harry.visitors.MutatingVisitor;
import harry.visitors.MutatingRowVisitor;
import harry.visitors.OperationExecutor;
-import harry.visitors.ParallelRecentValidator;
import harry.visitors.Visitor;
import harry.visitors.RecentValidator;
import harry.runner.Runner;
-import harry.visitors.Sampler;
import harry.util.BitSet;
+import org.reflections.Reflections;
+import org.reflections.scanners.Scanners;
+import org.reflections.util.NameHelper;
public class Configuration
{
+ private enum NameUtils implements NameHelper
+ {
+ INSTANCE;
+ }
+
private static final ObjectMapper mapper;
- private static final String DEFAULT_KEYSPACE = "harry";
+ private static <A extends Annotation> Set<Class<?>> findClassesMarkedWith(Class<A> annotation)
+ {
+ Reflections reflections = new Reflections(org.reflections.util.ConfigurationBuilder.build("harry").setExpandSuperTypes(false));
+ Collection<Class<?>> classes = NameUtils.INSTANCE.forNames(reflections.get(Scanners.TypesAnnotated.get(annotation.getName())),
+ reflections.getConfiguration().getClassLoaders());
+ return new HashSet<>(classes);
+ }
static
{
+
mapper = new ObjectMapper(new YAMLFactory()
.disable(YAMLGenerator.Feature.USE_NATIVE_TYPE_ID)
.disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER)
.disable(YAMLGenerator.Feature.CANONICAL_OUTPUT)
.enable(YAMLGenerator.Feature.INDENT_ARRAYS));
- mapper.registerSubtypes(Configuration.DebugApproximateMonotonicClockConfiguration.class);
- mapper.registerSubtypes(Configuration.ApproximateMonotonicClockConfiguration.class);
- mapper.registerSubtypes(Configuration.ConcurrentRunnerConfig.class);
- mapper.registerSubtypes(Configuration.SequentialRunnerConfig.class);
- mapper.registerSubtypes(Configuration.SingleVisitRunnerConfig.class);
- mapper.registerSubtypes(Configuration.DefaultDataTrackerConfiguration.class);
- mapper.registerSubtypes(Configuration.NoOpDataTrackerConfiguration.class);
-
- mapper.registerSubtypes(Configuration.QuiescentCheckerConfig.class);
- mapper.registerSubtypes(NoOpCheckerConfig.class);
- mapper.registerSubtypes(Configuration.DefaultCDSelectorConfiguration.class);
- mapper.registerSubtypes(Configuration.DefaultPDSelectorConfiguration.class);
- mapper.registerSubtypes(Configuration.ConstantDistributionConfig.class);
- mapper.registerSubtypes(DefaultSchemaProviderConfiguration.class);
- mapper.registerSubtypes(MutatingRowVisitorConfiguration.class);
-
- mapper.registerSubtypes(MutatingVisitorConfiguation.class);
- mapper.registerSubtypes(LoggingVisitorConfiguration.class);
- mapper.registerSubtypes(AllPartitionsValidatorConfiguration.class);
- mapper.registerSubtypes(ParallelRecentValidator.ParallelRecentValidatorConfig.class);
- mapper.registerSubtypes(Sampler.SamplerConfiguration.class);
- mapper.registerSubtypes(CorruptingVisitorConfiguration.class);
- mapper.registerSubtypes(RecentPartitionsValidatorConfiguration.class);
- mapper.registerSubtypes(FixedSchemaProviderConfiguration.class);
- mapper.registerSubtypes(AlwaysSamePartitionSelector.AlwaysSamePartitionSelectorConfiguration.class);
- mapper.registerSubtypes(OffsetClock.OffsetClockConfiguration.class);
- mapper.registerSubtypes(PrintlnSut.PrintlnSutConfiguration.class);
- mapper.registerSubtypes(NoOpDataTrackerConfiguration.class);
- mapper.registerSubtypes(NoOpMetricReporterConfiguration.class);
+
+ findClassesMarkedWith(JsonTypeName.class)
+ .forEach(mapper::registerSubtypes);
}
public final long seed;
@@ -156,6 +150,18 @@ public class Configuration
mapper.registerSubtypes(classes);
}
+ public static void toFile(File file, Configuration config)
+ {
+ try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file))))
+ {
+ bw.write(Configuration.toYamlString(config));
+ bw.flush();
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
public static String toYamlString(Configuration config)
{
try
@@ -229,21 +235,21 @@ public class Configuration
long seed = snapshot.seed;
- DataTracker tracker = snapshot.data_tracker == null ? new DefaultDataTrackerConfiguration().make() : snapshot.data_tracker.make();
OpSelectors.Rng rng = new OpSelectors.PCGFast(seed);
- OpSelectors.MonotonicClock clock = snapshot.clock.make();
-
- MetricReporter metricReporter = snapshot.metric_reporter.make();
-
// TODO: validate that operation kind is compatible with schema, due to statics etc
sut = snapshot.system_under_test.make();
-
SchemaSpec schemaSpec = snapshot.schema_provider.make(seed, sut);
schemaSpec.validate();
OpSelectors.PdSelector pdSelector = snapshot.partition_descriptor_selector.make(rng);
+ DataTrackerConfiguration dataTrackerConfiguration = snapshot.data_tracker == null ? new DefaultDataTrackerConfiguration() : snapshot.data_tracker;
+ DataTracker tracker = dataTrackerConfiguration.make(pdSelector, schemaSpec);
+
OpSelectors.DescriptorSelector descriptorSelector = snapshot.clustering_descriptor_selector.make(rng, schemaSpec);
+ OpSelectors.MonotonicClock clock = snapshot.clock.make();
+
+ MetricReporter metricReporter = snapshot.metric_reporter.make();
return new Run(rng,
clock,
@@ -306,7 +312,7 @@ public class Configuration
return this;
}
- public ConfigurationBuilder setDataTracker(String keyspace_ddl)
+ public ConfigurationBuilder setKeyspaceDdl(String keyspace_ddl)
{
this.keyspace_ddl = keyspace_ddl;
return this;
@@ -396,14 +402,18 @@ public class Configuration
{
ConfigurationBuilder builder = new ConfigurationBuilder();
builder.seed = seed;
+
builder.schema_provider = schema_provider;
+ builder.keyspace_ddl = keyspace_ddl;
builder.drop_schema = drop_schema;
builder.create_schema = create_schema;
builder.truncate_table = truncate_table;
+ builder.data_tracker = data_tracker;
builder.clock = clock;
builder.runner = runner;
builder.system_under_test = system_under_test;
+ builder.metric_reporter = metric_reporter;
builder.partition_descriptor_selector = partition_descriptor_selector;
builder.clustering_descriptor_selector = clustering_descriptor_selector;
@@ -426,7 +436,7 @@ public class Configuration
{
}
- public DataTracker make()
+ public DataTracker make(OpSelectors.PdSelector pdSelector, SchemaSpec schemaSpec)
{
return DataTracker.NO_OP;
}
@@ -437,28 +447,60 @@ public class Configuration
{
public final long max_seen_lts;
public final long max_complete_lts;
+ public final List<Long> reorder_buffer;
public DefaultDataTrackerConfiguration()
{
- this(-1, -1);
+ this(-1, -1, null);
}
@JsonCreator
public DefaultDataTrackerConfiguration(@JsonProperty(value = "max_seen_lts", defaultValue = "-1") long max_seen_lts,
- @JsonProperty(value = "max_complete_lts", defaultValue = "-1") long max_complete_lts)
+ @JsonProperty(value = "max_complete_lts", defaultValue = "-1") long max_complete_lts,
+ @JsonProperty(value = "reorder_buffer", defaultValue = "null") List<Long> reorder_buffer)
{
this.max_seen_lts = max_seen_lts;
this.max_complete_lts = max_complete_lts;
+ this.reorder_buffer = reorder_buffer;
}
- public DataTracker make()
+ @Override
+ public DataTracker make(OpSelectors.PdSelector pdSelector, SchemaSpec schemaSpec)
{
- DefaultDataTracker defaultDataTracker = new DefaultDataTracker();
- defaultDataTracker.forceLts(max_seen_lts, max_complete_lts);
- return defaultDataTracker;
+ DefaultDataTracker tracker = new DefaultDataTracker();
+ tracker.forceLts(max_seen_lts, max_complete_lts, reorder_buffer);
+ return tracker;
}
}
+ @JsonTypeName("locking")
+ public static class LockingDataTrackerConfiguration implements DataTrackerConfiguration
+ {
+ public final long max_seen_lts;
+ public final long max_complete_lts;
+ public final List<Long> reorder_buffer;
+
+ @JsonCreator
+ public LockingDataTrackerConfiguration(@JsonProperty(value = "max_seen_lts", defaultValue = "-1") long max_seen_lts,
+ @JsonProperty(value = "max_complete_lts", defaultValue = "-1") long max_complete_lts,
+ @JsonProperty(value = "reorder_buffer", defaultValue = "null") List<Long> reorder_buffer)
+ {
+ this.max_seen_lts = max_seen_lts;
+ this.max_complete_lts = max_complete_lts;
+ this.reorder_buffer = reorder_buffer;
+ }
+
+ @Override
+ public DataTracker make(OpSelectors.PdSelector pdSelector, SchemaSpec schemaSpec)
+ {
+ LockingDataTracker tracker = new LockingDataTracker(pdSelector, schemaSpec);
+ tracker.forceLts(max_seen_lts, max_complete_lts, reorder_buffer);
+ return tracker;
+ }
+ }
+
+
+
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT)
public interface ClockConfiguration extends OpSelectors.MonotonicClockFactory
{
@@ -538,22 +580,18 @@ public class Configuration
@JsonTypeName("concurrent")
public static class ConcurrentRunnerConfig implements RunnerConfiguration
{
- public final int concurrency;
-
- @JsonProperty(value = "visitors")
- public final List<VisitorConfiguration> visitorFactories;
+ @JsonProperty(value = "visitor_pools")
+ public final List<VisitorPoolConfiguration> visitor_pools;
public final long run_time;
public final TimeUnit run_time_unit;
@JsonCreator
- public ConcurrentRunnerConfig(@JsonProperty(value = "concurrency", defaultValue = "4") int concurrency,
- @JsonProperty(value = "visitors") List<VisitorConfiguration> visitors,
+ public ConcurrentRunnerConfig(@JsonProperty(value = "visitor_pools") List<VisitorPoolConfiguration> visitor_pools,
@JsonProperty(value = "run_time", defaultValue = "2") long runtime,
@JsonProperty(value = "run_time_unit", defaultValue = "HOURS") TimeUnit runtimeUnit)
{
- this.concurrency = concurrency;
- this.visitorFactories = visitors;
+ this.visitor_pools = visitor_pools;
this.run_time = runtime;
this.run_time_unit = runtimeUnit;
}
@@ -561,7 +599,29 @@ public class Configuration
@Override
public Runner make(Run run, Configuration config)
{
- return new Runner.ConcurrentRunner(run, config, concurrency, visitorFactories, run_time, run_time_unit);
+ return new Runner.ConcurrentRunner(run, config, visitor_pools, run_time, run_time_unit);
+ }
+ }
+
+ public static class VisitorPoolConfiguration
+ {
+ public final String prefix;
+ public final int concurrency;
+ public final VisitorConfiguration visitor;
+
+ @JsonCreator
+ public VisitorPoolConfiguration(@JsonProperty(value = "prefix") String prefix,
+ @JsonProperty(value = "concurrency") int concurrency,
+ @JsonProperty(value = "visitor") VisitorConfiguration visitor)
+ {
+ this.prefix = prefix;
+ this.concurrency = concurrency;
+ this.visitor = visitor;
+ }
+
+ public static VisitorPoolConfiguration pool(String prefix, int concurrency, VisitorConfiguration visitor)
+ {
+ return new VisitorPoolConfiguration(prefix, concurrency, visitor);
}
}
@@ -998,7 +1058,7 @@ public class Configuration
public Visitor make(Run run)
{
- return new AllPartitionsValidator(concurrency, trigger_after, run, modelConfiguration);
+ return new AllPartitionsValidator(run, concurrency, trigger_after, modelConfiguration);
}
}
diff --git a/harry-core/src/harry/core/Run.java b/harry-core/src/harry/core/Run.java
index e3bc203..be47d45 100644
--- a/harry-core/src/harry/core/Run.java
+++ b/harry-core/src/harry/core/Run.java
@@ -38,7 +38,6 @@ public class Run
public final MetricReporter metricReporter;
-
public Run(OpSelectors.Rng rng,
OpSelectors.MonotonicClock clock,
OpSelectors.PdSelector pdSelector,
@@ -63,7 +62,6 @@ public class Run
SystemUnderTest sut,
MetricReporter metricReporter)
{
-
this.rng = rng;
this.clock = clock;
this.pdSelector = pdSelector;
diff --git a/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java b/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java
index ace55b4..049c575 100644
--- a/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java
+++ b/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java
@@ -31,6 +31,7 @@ import harry.model.SelectHelper;
import harry.model.sut.SystemUnderTest;
import harry.operations.WriteHelper;
import harry.operations.Query;
+import harry.runner.DataTracker;
public class AddExtraRowCorruptor implements QueryResponseCorruptor
{
@@ -38,34 +39,34 @@ public class AddExtraRowCorruptor implements QueryResponseCorruptor
private final SchemaSpec schema;
private final OpSelectors.MonotonicClock clock;
+ private final DataTracker tracker;
private final OpSelectors.DescriptorSelector descriptorSelector;
public AddExtraRowCorruptor(SchemaSpec schema,
OpSelectors.MonotonicClock clock,
+ DataTracker tracker,
OpSelectors.DescriptorSelector descriptorSelector)
{
this.schema = schema;
this.clock = clock;
+ this.tracker = tracker;
this.descriptorSelector = descriptorSelector;
}
public boolean maybeCorrupt(Query query, SystemUnderTest sut)
{
Set<Long> cds = new HashSet<>();
- long maxLts = 0;
+ long maxLts = tracker.maxStarted();
for (Object[] obj : sut.execute(query.toSelectStatement(), SystemUnderTest.ConsistencyLevel.ALL))
{
ResultSetRow row = SelectHelper.resultSetToRow(schema, clock, obj);
// TODO: extract CD cheaper
cds.add(row.cd);
- for (int i = 0; i < row.lts.length; i++)
- maxLts = Math.max(maxLts, row.lts[i]);
}
-
boolean partitionIsFull = cds.size() >= descriptorSelector.maxPartitionSize();
long attempt = 0;
- long cd = descriptorSelector.randomCd(query.pd, attempt, schema);;
+ long cd = descriptorSelector.randomCd(query.pd, attempt, schema);
while (!query.match(cd) || cds.contains(cd))
{
if (partitionIsFull)
diff --git a/harry-core/src/harry/data/ResultSetRow.java b/harry-core/src/harry/data/ResultSetRow.java
index 48cfc03..d6ff77b 100644
--- a/harry-core/src/harry/data/ResultSetRow.java
+++ b/harry-core/src/harry/data/ResultSetRow.java
@@ -21,6 +21,7 @@ package harry.data;
import java.util.Arrays;
import harry.ddl.SchemaSpec;
+import harry.util.StringUtils;
public class ResultSetRow
{
@@ -52,35 +53,22 @@ public class ResultSetRow
return "resultSetRow("
+ pd +
"L, " + cd +
- (sds == null ? "" : "L, values(" + toString(sds) + ")") +
- (slts == null ? "" : ", lts(" + toString(slts) + ")") +
- ", values(" + toString(vds) + ")" +
- ", lts(" + toString(lts) + ")" +
+ (sds == null ? "" : "L, values(" + StringUtils.toString(sds) + ")") +
+ (slts == null ? "" : ", lts(" + StringUtils.toString(slts) + ")") +
+ ", values(" + StringUtils.toString(vds) + ")" +
+ ", lts(" + StringUtils.toString(lts) + ")" +
")";
}
- public String toString(long[] arr)
- {
- String s = "";
- for (int i = 0; i < arr.length; i++)
- {
- s += arr[i];
- s += "L";
- if (i < (arr.length - 1))
- s += ',';
- }
- return s;
- }
-
public String toString(SchemaSpec schema)
{
return "resultSetRow("
+ pd +
"L, " + cd +
- (sds == null ? "" : "L, staticValues(" + toString(sds) + ")") +
- (slts == null ? "" : ", slts(" + toString(slts) + ")") +
- ", values(" + toString(vds) + ")" +
- ", lts(" + toString(lts) + ")" +
+ (sds == null ? "" : "L, staticValues(" + StringUtils.toString(sds) + ")") +
+ (slts == null ? "" : ", slts(" + StringUtils.toString(slts) + ")") +
+ ", values(" + StringUtils.toString(vds) + ")" +
+ ", lts(" + StringUtils.toString(lts) + ")" +
", clustering=" + Arrays.toString(schema.inflateClusteringKey(cd)) +
", values=" + Arrays.toString(schema.inflateRegularColumns(vds)) +
(sds == null ? "" : ", statics=" + Arrays.toString(schema.inflateStaticColumns(sds))) +
diff --git a/harry-core/src/harry/ddl/SchemaGenerators.java b/harry-core/src/harry/ddl/SchemaGenerators.java
index f56c0d5..33cf08a 100644
--- a/harry-core/src/harry/ddl/SchemaGenerators.java
+++ b/harry-core/src/harry/ddl/SchemaGenerators.java
@@ -61,7 +61,8 @@ public class SchemaGenerators
ColumnSpec.int16Type,
ColumnSpec.int32Type,
ColumnSpec.int64Type,
- ColumnSpec.booleanType,
+ // TODO: ByteBufferUtil#objectToBytes doesn't understand booleans
+ //ColumnSpec.booleanType,
ColumnSpec.floatType,
ColumnSpec.doubleType,
ColumnSpec.asciiType,
@@ -354,7 +355,7 @@ public class SchemaGenerators
{
return new SchemaGenerators.Builder(DEFAULT_KEYSPACE_NAME, () -> table)
.partitionKeySpec(1, 3,
- columnTypes)
+ partitionKeyTypes)
.clusteringKeySpec(1, 3,
clusteringKeyTypes)
.regularColumnSpec(3, 5,
diff --git a/harry-core/src/harry/dsl/HistoryBuilder.java b/harry-core/src/harry/dsl/HistoryBuilder.java
index fa26277..06e09de 100644
--- a/harry-core/src/harry/dsl/HistoryBuilder.java
+++ b/harry-core/src/harry/dsl/HistoryBuilder.java
@@ -115,6 +115,11 @@ public class HistoryBuilder implements Iterable<ReplayingVisitor.Visit>
{
return historyBuilder.position(pd(lts));
}
+
+ public long maxPosition(long maxLts)
+ {
+ return historyBuilder.partitions;
+ }
}
private static abstract class Step
diff --git a/harry-core/src/harry/model/AlwaysSamePartitionSelector.java b/harry-core/src/harry/model/AlwaysSamePartitionSelector.java
index 43160fb..3069d9a 100644
--- a/harry-core/src/harry/model/AlwaysSamePartitionSelector.java
+++ b/harry-core/src/harry/model/AlwaysSamePartitionSelector.java
@@ -51,6 +51,11 @@ public class AlwaysSamePartitionSelector extends OpSelectors.PdSelector
return 0;
}
+ public long maxPosition(long maxLts)
+ {
+ return 0;
+ }
+
@JsonTypeName("always_same")
public static class AlwaysSamePartitionSelectorConfiguration implements Configuration.PDSelectorConfiguration
{
diff --git a/harry-core/src/harry/model/OpSelectors.java b/harry-core/src/harry/model/OpSelectors.java
index 7727828..b9bb34c 100644
--- a/harry-core/src/harry/model/OpSelectors.java
+++ b/harry-core/src/harry/model/OpSelectors.java
@@ -18,11 +18,9 @@
package harry.model;
-import java.util.ArrayList;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
-import java.util.function.LongConsumer;
import harry.core.Configuration;
import harry.core.VisibleForTesting;
@@ -124,6 +122,7 @@ public interface OpSelectors
// TODO: right now, we can only calculate a position for 64-bit (in other words, full entropy) pds
public abstract long positionFor(long lts);
+ public abstract long maxPosition(long maxLts);
}
public static interface PdSelectorFactory
@@ -298,10 +297,31 @@ public interface OpSelectors
public long minLtsFor(long pd)
{
- long position = rng.sequenceNumber(pd, PARTITION_DESCRIPTOR_STREAM_ID);
+ long position = positionForPd(pd);
return minLtsAt(position);
}
+ public long randomVisitedPd(long maxLts, long visitLts, SchemaSpec schema)
+ {
+ long maxPosition = maxPosition(maxLts);
+ if (maxPosition == 0)
+ return schema.adjustPdEntropy(rng.randomNumber(0, PARTITION_DESCRIPTOR_STREAM_ID));
+
+ int idx = RngUtils.asInt(rng.randomNumber(visitLts, maxPosition), 0, (int) (maxPosition - 1));
+ return schema.adjustPdEntropy(rng.randomNumber(idx, PARTITION_DESCRIPTOR_STREAM_ID));
+ }
+
+ public long maxPosition(long maxLts)
+ {
+ long timesCycled = maxLts / switchAfter;
+ long windowStart = timesCycled * windowSize;
+
+ // We have cycled through _current_ window at least once
+ if (maxLts > (windowStart * slideAfterRepeats) + windowSize)
+ return windowStart + windowSize;
+ return windowStart + maxLts % windowSize;
+ }
+
// TODO: add maxPosition to make it easier/more accessible for the components like sampler, etc
public long positionFor(long lts)
{
diff --git a/harry-integration/src/harry/runner/QueryingNoOpChecker.java b/harry-core/src/harry/model/QueryingNoOpValidator.java
similarity index 80%
rename from harry-integration/src/harry/runner/QueryingNoOpChecker.java
rename to harry-core/src/harry/model/QueryingNoOpValidator.java
index 0e7d4fb..a2da871 100644
--- a/harry-integration/src/harry/runner/QueryingNoOpChecker.java
+++ b/harry-core/src/harry/model/QueryingNoOpValidator.java
@@ -16,27 +16,26 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.model;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonTypeName;
import harry.core.Configuration;
import harry.core.Run;
-import harry.model.Model;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
import harry.operations.Query;
-public class QueryingNoOpChecker implements Model
+/**
+ * A model that can be used to "simply" run random queries. Does not perform validation
+ * of the results that it sees. Useful for increasing concurrency and triggering
+ * exceptions rather than data loss issues.
+ */
+public class QueryingNoOpValidator implements Model
{
- public static void init()
- {
- Configuration.registerSubtypes(QueryingNoOpCheckerConfig.class);
- }
-
private final Run run;
- public QueryingNoOpChecker(Run run)
+ public QueryingNoOpValidator(Run run)
{
this.run = run;
}
@@ -50,7 +49,7 @@ public class QueryingNoOpChecker implements Model
compiled.bindings());
}
- @JsonTypeName("querying_no_op_checker")
+ @JsonTypeName("no_op")
public static class QueryingNoOpCheckerConfig implements Configuration.ModelConfiguration
{
@JsonCreator
@@ -60,7 +59,7 @@ public class QueryingNoOpChecker implements Model
public Model make(Run run)
{
- return new QueryingNoOpChecker(run);
+ return new QueryingNoOpValidator(run);
}
}
}
diff --git a/harry-core/src/harry/model/QuiescentChecker.java b/harry-core/src/harry/model/QuiescentChecker.java
index 0246e98..0638915 100644
--- a/harry-core/src/harry/model/QuiescentChecker.java
+++ b/harry-core/src/harry/model/QuiescentChecker.java
@@ -28,9 +28,9 @@ import harry.core.Run;
import harry.data.ResultSetRow;
import harry.ddl.SchemaSpec;
import harry.model.sut.SystemUnderTest;
+import harry.operations.Query;
import harry.reconciler.Reconciler;
import harry.runner.DataTracker;
-import harry.operations.Query;
import static harry.generators.DataGenerators.NIL_DESCR;
@@ -59,22 +59,17 @@ public class QuiescentChecker implements Model
public void validate(Query query)
{
+ tracker.beginValidation(query.pd);
validate(() -> SelectHelper.execute(sut, clock, query), query);
+ tracker.endValidation(query.pd);
}
protected void validate(Supplier<List<ResultSetRow>> rowsSupplier, Query query)
{
- long maxCompeteLts = tracker.maxConsecutiveFinished();
- long maxSeenLts = tracker.maxStarted();
-
- assert maxCompeteLts == maxSeenLts : String.format("Runner hasn't settled down yet. " +
- "Quiescent model can't be reliably used in such cases. " +
- "Max complete: %d. Max seen: %d",
- maxCompeteLts, maxSeenLts);
-
List<ResultSetRow> actualRows = rowsSupplier.get();
Iterator<ResultSetRow> actual = actualRows.iterator();
- Reconciler.PartitionState partitionState = reconciler.inflatePartitionState(query.pd, maxSeenLts, query);
+
+ Reconciler.PartitionState partitionState = reconciler.inflatePartitionState(query.pd, tracker, query);
Collection<Reconciler.RowState> expectedRows = partitionState.rows(query.reverse);
Iterator<Reconciler.RowState> expected = expectedRows.iterator();
@@ -85,7 +80,7 @@ public class QuiescentChecker implements Model
ResultSetRow actualRowState = actual.next();
if (actualRowState.cd != partitionState.staticRow().cd)
throw new ValidationException(partitionState.toString(schema),
- toString(actualRows, schema),
+ toString(actualRows),
"Found a row while model predicts statics only:" +
"\nExpected: %s" +
"\nActual: %s" +
@@ -97,10 +92,9 @@ public class QuiescentChecker implements Model
{
if (actualRowState.vds[i] != NIL_DESCR || actualRowState.lts[i] != NO_TIMESTAMP)
throw new ValidationException(partitionState.toString(schema),
- toString(actualRows, schema),
+ toString(actualRows),
"Found a row while model predicts statics only:" +
"\nActual: %s" +
- "\nQuery: %s" +
"\nQuery: %s",
actualRowState, query.toSelectStatement());
}
@@ -115,7 +109,7 @@ public class QuiescentChecker implements Model
// TODO: this is not necessarily true. It can also be that ordering is incorrect.
if (actualRowState.cd != expectedRowState.cd)
throw new ValidationException(partitionState.toString(schema),
- toString(actualRows, schema),
+ toString(actualRows),
"Found a row in the model that is not present in the resultset:" +
"\nExpected: %s" +
"\nActual: %s" +
@@ -125,7 +119,7 @@ public class QuiescentChecker implements Model
if (!Arrays.equals(actualRowState.vds, expectedRowState.vds))
throw new ValidationException(partitionState.toString(schema),
- toString(actualRows, schema),
+ toString(actualRows),
"Returned row state doesn't match the one predicted by the model:" +
"\nExpected: %s (%s)" +
"\nActual: %s (%s)." +
@@ -136,11 +130,12 @@ public class QuiescentChecker implements Model
if (!Arrays.equals(actualRowState.lts, expectedRowState.lts))
throw new ValidationException(partitionState.toString(schema),
- toString(actualRows, schema),
+ toString(actualRows),
"Timestamps in the row state don't match ones predicted by the model:" +
"\nExpected: %s (%s)" +
"\nActual: %s (%s)." +
- "\nQuery: %s",
+ "\nQuery: %s" +
+ "\nMax started: %d, Max finished: %d, %d reordered: %s",
Arrays.toString(expectedRowState.lts), expectedRowState.toString(schema),
Arrays.toString(actualRowState.lts), actualRowState,
query.toSelectStatement());
@@ -152,7 +147,7 @@ public class QuiescentChecker implements Model
if (actual.hasNext() || expected.hasNext())
{
throw new ValidationException(partitionState.toString(schema),
- toString(actualRows, schema),
+ toString(actualRows),
"Expected results to have the same number of results, but %s result iterator has more results." +
"\nExpected: %s" +
"\nActual: %s" +
@@ -173,7 +168,7 @@ public class QuiescentChecker implements Model
{
if (!Arrays.equals(staticRow.vds, actualRowState.sds))
throw new ValidationException(partitionState.toString(schemaSpec),
- toString(actualRows, schemaSpec),
+ toString(actualRows),
"Returned static row state doesn't match the one predicted by the model:" +
"\nExpected: %s (%s)" +
"\nActual: %s (%s)." +
@@ -184,7 +179,7 @@ public class QuiescentChecker implements Model
if (!Arrays.equals(staticRow.lts, actualRowState.slts))
throw new ValidationException(partitionState.toString(schemaSpec),
- toString(actualRows, schemaSpec),
+ toString(actualRows),
"Timestamps in the static row state don't match ones predicted by the model:" +
"\nExpected: %s (%s)" +
"\nActual: %s (%s)." +
@@ -203,12 +198,12 @@ public class QuiescentChecker implements Model
return builder.toString();
}
- public static String toString(List<ResultSetRow> collection, SchemaSpec schema)
+ public static String toString(List<ResultSetRow> collection)
{
StringBuilder builder = new StringBuilder();
for (ResultSetRow rowState : collection)
- builder.append(rowState.toString(schema)).append("\n");
+ builder.append(rowState.toString()).append("\n");
return builder.toString();
}
-}
\ No newline at end of file
+}
diff --git a/harry-core/src/harry/model/QuiescentLocalStateCheckerBase.java b/harry-core/src/harry/model/QuiescentLocalStateCheckerBase.java
new file mode 100644
index 0000000..de4ce85
--- /dev/null
+++ b/harry-core/src/harry/model/QuiescentLocalStateCheckerBase.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.model;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NavigableMap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import harry.core.Run;
+import harry.data.ResultSetRow;
+import harry.model.sut.SystemUnderTest;
+import harry.operations.CompiledStatement;
+import harry.operations.Query;
+
+import static harry.model.SelectHelper.resultSetToRow;
+import static harry.model.sut.TokenPlacementModel.Node;
+import static harry.model.sut.TokenPlacementModel.Range;
+import static harry.model.sut.TokenPlacementModel.getReplicas;
+
+public abstract class QuiescentLocalStateCheckerBase extends QuiescentChecker
+{
+ private static final Logger logger = LoggerFactory.getLogger(QuiescentLocalStateCheckerBase.class);
+
+ public final SystemUnderTest sut;
+ public final int rf;
+ private final OpSelectors.PdSelector pdSelector;
+
+ public QuiescentLocalStateCheckerBase(Run run)
+ {
+ this(run, 3);
+ }
+
+ public QuiescentLocalStateCheckerBase(Run run, int rf)
+ {
+ super(run);
+ this.sut = run.sut;
+ this.rf = rf;
+ this.pdSelector = run.pdSelector;
+ }
+
+ @SuppressWarnings("unused")
+ public void validateAll()
+ {
+ NavigableMap<Range, List<Node>> ring = getRing();
+
+ for (int lts = 0; lts < clock.peek(); lts++)
+ validate(Query.selectPartition(schema, pdSelector.pd(lts, schema), false), ring);
+ }
+
+ @Override
+ public void validate(Query query)
+ {
+ NavigableMap<Range, List<Node>> ring = getRing();
+ tracker.beginValidation(query.pd);
+ validate(query, ring);
+ tracker.endValidation(query.pd);
+ }
+
+ protected void validate(Query query, NavigableMap<Range, List<Node>> ring)
+ {
+ CompiledStatement compiled = query.toSelectStatement();
+ List<Node> replicas = getReplicas(ring, token(query.pd));
+
+ logger.trace("Predicted {} as replicas for {}. Ring: {}", replicas, query.pd, ring);
+ for (Node node : replicas)
+ {
+ try
+ {
+ validate(() -> {
+ Object[][] objects = executeNodeLocal(compiled.cql(), node, compiled.bindings());
+
+ List<ResultSetRow> result = new ArrayList<>();
+ for (Object[] obj : objects)
+ result.add(resultSetToRow(query.schemaSpec, clock, obj));
+
+ return result;
+ }, query);
+ }
+ catch (ValidationException e)
+ {
+ throw new AssertionError(String.format("Caught error while validating replica %s of replica set %s",
+ node, replicas),
+ e);
+ }
+ }
+ }
+
+ protected abstract NavigableMap<Range, List<Node>> getRing();
+ protected abstract long token(long pd);
+ protected abstract Object[][] executeNodeLocal(String statement, Node node, Object... bindings);
+}
diff --git a/harry-integration/src/harry/runner/RepairingLocalStateValidator.java b/harry-core/src/harry/model/RepairingLocalStateValidator.java
similarity index 50%
rename from harry-integration/src/harry/runner/RepairingLocalStateValidator.java
rename to harry-core/src/harry/model/RepairingLocalStateValidator.java
index 71c1535..ff93b29 100644
--- a/harry-integration/src/harry/runner/RepairingLocalStateValidator.java
+++ b/harry-core/src/harry/model/RepairingLocalStateValidator.java
@@ -16,42 +16,43 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.model;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.function.LongPredicate;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import harry.core.Configuration;
import harry.core.Run;
-import harry.data.ResultSetRow;
-import harry.model.Model;
-import harry.model.OpSelectors;
-import harry.model.QuiescentChecker;
-import harry.model.sut.InJvmSut;
-import harry.model.sut.SystemUnderTest;
-import harry.operations.CompiledStatement;
-import harry.operations.Query;
+import harry.model.sut.injvm.InJvmSut;
import harry.visitors.AllPartitionsValidator;
import harry.visitors.Visitor;
-import static harry.model.SelectHelper.resultSetToRow;
-
+/**
+ * Validator similar to {@link AllPartitionsValidator}, but performs
+ * repair before checking node states.
+ *
+ * Can be useful for testing repair, bootstrap, and streaming code.
+ */
public class RepairingLocalStateValidator extends AllPartitionsValidator
{
- public static void init()
+ private final InJvmSut inJvmSut;
+ private final OpSelectors.MonotonicClock clock;
+
+ public static Configuration.VisitorConfiguration factory(int concurrency, LongPredicate condition, Model.ModelFactory modelFactory)
{
- Configuration.registerSubtypes(RepairingLocalStateValidatorConfiguration.class,
- QuiescentCheckerConfig.class);
+ return (r) -> new RepairingLocalStateValidator(r, concurrency, condition, modelFactory);
}
- private final InJvmSut inJvmSut;
- private final OpSelectors.MonotonicClock clock;
- public RepairingLocalStateValidator(int concurrency, int triggerAfter, Run run, Model.ModelFactory modelFactory)
+ public RepairingLocalStateValidator(Run run, int concurrency, long triggerAfter, Model.ModelFactory modelFactory)
{
- super(concurrency, triggerAfter, run, modelFactory);
+ this(run, concurrency, (lts) -> lts > 0 && lts % triggerAfter == 0, modelFactory);
+ }
+
+ public RepairingLocalStateValidator(Run run, int concurrency, LongPredicate condition, Model.ModelFactory modelFactory)
+ {
+ super(run, concurrency, condition, modelFactory);
this.inJvmSut = (InJvmSut) run.sut;
this.clock = run.clock;
}
@@ -59,12 +60,10 @@ public class RepairingLocalStateValidator extends AllPartitionsValidator
public void visit()
{
long lts = clock.peek();
- if (lts > 0 && lts % triggerAfter == 0)
+ if (condition.test(lts))
{
System.out.println("Starting repair...");
-
inJvmSut.cluster().stream().forEach((instance) -> instance.nodetool("repair", "--full"));
-
System.out.println("Validating partitions...");
super.visit();
}
@@ -89,55 +88,7 @@ public class RepairingLocalStateValidator extends AllPartitionsValidator
public Visitor make(Run run)
{
- return new RepairingLocalStateValidator(concurrency, trigger_after, run, modelConfiguration);
- }
- }
-
- public static class QuiescentLocalStateChecker extends QuiescentChecker
- {
- public final InJvmSut inJvmSut;
-
- public QuiescentLocalStateChecker(Run run)
- {
- super(run);
- assert run.sut instanceof InJvmSut;
-
- this.inJvmSut = (InJvmSut) run.sut;
- }
-
- @Override
- public void validate(Query query)
- {
- CompiledStatement compiled = query.toSelectStatement();
- int[] replicas = inJvmSut.getReplicasFor(schema.inflatePartitionKey(query.pd), schema.keyspace, schema.table);
- for (int node : replicas)
- {
- validate(() -> {
- Object[][] objects = inJvmSut.execute(compiled.cql(),
- SystemUnderTest.ConsistencyLevel.NODE_LOCAL,
- node,
- compiled.bindings());
- List<ResultSetRow> result = new ArrayList<>();
- for (Object[] obj : objects)
- result.add(resultSetToRow(query.schemaSpec, clock, obj));
-
- return result;
- }, query);
- }
- }
- }
-
- @JsonTypeName("quiescent_local_state_checker")
- public static class QuiescentCheckerConfig implements Configuration.ModelConfiguration
- {
- @JsonCreator
- public QuiescentCheckerConfig()
- {
- }
-
- public Model make(Run run)
- {
- return new QuiescentLocalStateChecker(run);
+ return new RepairingLocalStateValidator(run, concurrency, trigger_after, modelConfiguration);
}
}
}
diff --git a/harry-core/src/harry/model/clock/OffsetClock.java b/harry-core/src/harry/model/clock/OffsetClock.java
index 6e8f891..2ea3576 100644
--- a/harry-core/src/harry/model/clock/OffsetClock.java
+++ b/harry-core/src/harry/model/clock/OffsetClock.java
@@ -28,12 +28,18 @@ import harry.model.OpSelectors;
public class OffsetClock implements OpSelectors.MonotonicClock
{
- final AtomicLong lts = new AtomicLong(ApproximateMonotonicClock.START_VALUE);
+ final AtomicLong lts;
private final long base;
public OffsetClock(long base)
{
+ this(ApproximateMonotonicClock.START_VALUE, base);
+ }
+
+ public OffsetClock(long startValue, long base)
+ {
+ this.lts = new AtomicLong(startValue);
this.base = base;
}
@@ -59,23 +65,26 @@ public class OffsetClock implements OpSelectors.MonotonicClock
public Configuration.ClockConfiguration toConfig()
{
- throw new RuntimeException("not implemented");
+ return new OffsetClockConfiguration(lts.get(), base);
}
@JsonTypeName("offset")
public static class OffsetClockConfiguration implements Configuration.ClockConfiguration
{
public final long offset;
+ public final long base;
@JsonCreator
- public OffsetClockConfiguration(@JsonProperty("offset") int offset)
+ public OffsetClockConfiguration(@JsonProperty("offset") long offset,
+ @JsonProperty(value = "base", defaultValue = "0") long base)
{
this.offset = offset;
+ this.base = base;
}
public OpSelectors.MonotonicClock make()
{
- return new OffsetClock(offset);
+ return new OffsetClock(base, offset);
}
}
}
diff --git a/harry-core/src/harry/model/sut/TokenPlacementModel.java b/harry-core/src/harry/model/sut/TokenPlacementModel.java
new file mode 100644
index 0000000..fe161f9
--- /dev/null
+++ b/harry-core/src/harry/model/sut/TokenPlacementModel.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.model.sut;
+
+import java.net.InetAddress;
+import java.util.*;
+
+public class TokenPlacementModel
+{
+ /**
+ * Replicate ranges to rf nodes.
+ */
+ public static NavigableMap<Range, List<Node>> replicate(List<Node> nodes, int rf)
+ {
+ nodes.sort(Node::compareTo);
+ List<Range> ranges = toRanges(nodes);
+ return replicate(ranges, nodes, rf);
+ }
+
+ public static List<Node> getReplicas(NavigableMap<TokenPlacementModel.Range, List<TokenPlacementModel.Node>> ring, long token)
+ {
+ return ring.get(ring.floorKey(new Range(token, token, true)));
+ }
+
+ public static NavigableMap<Range, List<Node>> replicate(List<Range> ranges, List<Node> nodes, int rf)
+ {
+ NavigableMap<Range, List<Node>> replication = new TreeMap<>();
+ for (Range range : ranges)
+ {
+ Set<String> names = new HashSet<>();
+ List<Node> replicas = new ArrayList<>();
+ int idx = primaryReplica(nodes, range);
+ if (idx >= 0)
+ {
+ for (int i = idx; i < nodes.size() && replicas.size() < rf; i++)
+ addIfUnique(replicas, names, nodes.get(i));
+
+ for (int i = 0; replicas.size() < rf && i < idx; i++)
+ addIfUnique(replicas, names, nodes.get(i));
+ if (range.start == Long.MIN_VALUE)
+ replication.put(ranges.get(ranges.size() - 1), replicas);
+ replication.put(range, replicas);
+ }
+ else
+ {
+ // if the range end is larger than the highest assigned token, then treat it
+ // as part of the wraparound and replicate it to the same nodes as the first
+ // range. This is most likely caused by a decommission removing the node with
+ // the largest token.
+ replication.put(range, replication.get(ranges.get(0)));
+ }
+ }
+ return replication;
+ }
+
+ public static void addIfUnique(List<Node> nodes, Set<String> names, Node node)
+ {
+ if (names.contains(node.id))
+ return;
+ nodes.add(node);
+ names.add(node.id);
+ }
+
+ /**
+ * Finds a primary replica
+ */
+ public static int primaryReplica(List<Node> nodes, Range range)
+ {
+ for (int i = 0; i < nodes.size(); i++)
+ {
+ if (range.end != Long.MIN_VALUE && nodes.get(i).token >= range.end)
+ return i;
+ }
+ return -1;
+ }
+
+ /**
+ * Generates token ranges from the list of nodes
+ */
+ public static List<Range> toRanges(List<Node> nodes)
+ {
+ List<Long> tokens = new ArrayList<>();
+ for (Node node : nodes)
+ {
+ if (node.token != Long.MIN_VALUE)
+ tokens.add(node.token);
+ }
+ tokens.add(Long.MIN_VALUE);
+ tokens.sort(Long::compareTo);
+
+ List<Range> ranges = new ArrayList<>(tokens.size() + 1);
+ long prev = tokens.get(0);
+ for (int i = 1; i < tokens.size(); i++)
+ {
+ long current = tokens.get(i);
+ ranges.add(new Range(prev, current));
+ prev = current;
+ }
+ ranges.add(new Range(prev, Long.MIN_VALUE));
+ return Collections.unmodifiableList(ranges);
+ }
+
+ public static List<Node> peerStateToNodes(Object[][] resultset)
+ {
+ List<Node> nodes = new ArrayList<>();
+ for (Object[] row : resultset)
+ {
+ InetAddress address = (InetAddress) row[0];
+ Set<String> tokens = (Set<String>) row[1];
+ for (String token : tokens)
+ nodes.add(new Node(Long.parseLong(token), address.toString()));
+ }
+ return nodes;
+ }
+
+ public static class Range implements Comparable<Range>
+ {
+ public final long start;
+ public final long end;
+
+ public Range(long start, long end)
+ {
+ this(start, end, false);
+ }
+
+ public Range(long start, long end, boolean skipAssert)
+ {
+ assert skipAssert || (end > start || end == Long.MIN_VALUE);
+ this.start = start;
+ this.end = end;
+ }
+
+ public boolean contains(long token)
+ {
+ return token > start && (token <= end || end == Long.MIN_VALUE);
+ }
+
+ public int compareTo(Range o)
+ {
+ int res = Long.compare(start, o.start);
+ if (res == 0)
+ return Long.compare(end, o.end);
+ return res;
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Range range = (Range) o;
+ return start == range.start && end == range.end;
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(start, end);
+ }
+
+ public String toString()
+ {
+ return "(" +
+ "" + (start == Long.MIN_VALUE ? "MIN" : start) +
+ ", " + (end == Long.MAX_VALUE ? "MAX" : end) +
+ ']';
+ }
+ }
+
+ public static class Node implements Comparable<Node>
+ {
+ public final long token;
+ public final String id;
+
+ public Node(long token, String id)
+ {
+ this.token = token;
+ this.id = id;
+ }
+
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Node node = (Node) o;
+ return Objects.equals(id, node.id);
+ }
+
+ public int hashCode()
+ {
+ return Objects.hash(id);
+ }
+
+ public int compareTo(Node o)
+ {
+ return Long.compare(token, o.token);
+ }
+
+ public String toString()
+ {
+ return "" + id + "@" + token;
+ }
+ }
+}
diff --git a/harry-integration/src/harry/model/sut/InJVMTokenAwareVisitExecutor.java b/harry-core/src/harry/model/sut/injvm/InJVMTokenAwareVisitExecutor.java
similarity index 53%
rename from harry-integration/src/harry/model/sut/InJVMTokenAwareVisitExecutor.java
rename to harry-core/src/harry/model/sut/injvm/InJVMTokenAwareVisitExecutor.java
index 6bc0705..17a91d7 100644
--- a/harry-integration/src/harry/model/sut/InJVMTokenAwareVisitExecutor.java
+++ b/harry-core/src/harry/model/sut/injvm/InJVMTokenAwareVisitExecutor.java
@@ -16,43 +16,61 @@
* limitations under the License.
*/
-package harry.model.sut;
+package harry.model.sut.injvm;
-import java.util.Random;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NavigableMap;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import harry.core.Run;
import harry.ddl.SchemaSpec;
+import harry.model.sut.SystemUnderTest;
+import harry.model.sut.TokenPlacementModel;
import harry.operations.CompiledStatement;
+import harry.util.ByteUtils;
+import harry.util.TokenUtil;
import harry.visitors.GeneratingVisitor;
import harry.visitors.LoggingVisitor;
import harry.visitors.OperationExecutor;
+import harry.visitors.VisitExecutor;
import harry.visitors.Visitor;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.IInstance;
+
+import static harry.model.sut.TokenPlacementModel.peerStateToNodes;
+import static harry.model.sut.TokenPlacementModel.getReplicas;
public class InJVMTokenAwareVisitExecutor extends LoggingVisitor.LoggingVisitorExecutor
{
- public static void init()
- {
- harry.core.Configuration.registerSubtypes(Configuration.class);
- }
-
private final InJvmSut sut;
+ private final int rf;
private final SystemUnderTest.ConsistencyLevel cl;
private final SchemaSpec schema;
- private final int maxRetries = 10;
+ private final int MAX_RETRIES = 10;
+
+ public static Function<Run, VisitExecutor> factory(OperationExecutor.RowVisitorFactory rowVisitorFactory,
+ SystemUnderTest.ConsistencyLevel cl,
+ int rf)
+ {
+ return (run) -> new InJVMTokenAwareVisitExecutor(run, rowVisitorFactory, cl, rf);
+ }
public InJVMTokenAwareVisitExecutor(Run run,
OperationExecutor.RowVisitorFactory rowVisitorFactory,
- SystemUnderTest.ConsistencyLevel cl)
+ SystemUnderTest.ConsistencyLevel cl,
+ int rf)
{
super(run, rowVisitorFactory.make(run));
this.sut = (InJvmSut) run.sut;
this.schema = run.schemaSpec;
this.cl = cl;
+ this.rf = rf;
}
@Override
@@ -66,20 +84,26 @@ public class InJVMTokenAwareVisitExecutor extends LoggingVisitor.LoggingVisitorE
if (sut.isShutdown())
throw new IllegalStateException("System under test is shut down");
- if (retries > this.maxRetries)
+ if (retries > this.MAX_RETRIES)
throw new IllegalStateException(String.format("Can not execute statement %s after %d retries", statement, retries));
- Object[] partitionKey = schema.inflatePartitionKey(pd);
- int[] replicas = sut.getReplicasFor(partitionKey, schema.keyspace, schema.table);
- // TODO: find a better source of entropy
- int replica = replicas[new Random(lts).nextInt(replicas.length)];
+ Object[] pk = schema.inflatePartitionKey(pd);
+ List<TokenPlacementModel.Node> replicas = getReplicas(getRing(), TokenUtil.token(ByteUtils.compose(ByteUtils.objectsToBytes(pk))));
+
+ TokenPlacementModel.Node replica = replicas.get((int) (lts % replicas.size()));
if (cl == SystemUnderTest.ConsistencyLevel.NODE_LOCAL)
{
- future.complete(sut.cluster.get(replica).executeInternal(statement.cql(), statement.bindings()));
+ future.complete(executeNodeLocal(statement.cql(), replica, statement.bindings()));
}
else
{
- CompletableFuture.supplyAsync(() -> sut.cluster.coordinator(replica).execute(statement.cql(), InJvmSut.toApiCl(cl), statement.bindings()), executor)
+ CompletableFuture.supplyAsync(() -> sut.cluster
+ .stream()
+ .filter((n) -> n.config().broadcastAddress().toString().contains(replica.id))
+ .findFirst()
+ .get()
+ .coordinator()
+ .execute(statement.cql(), InJvmSut.toApiCl(cl), statement.bindings()), executor)
.whenComplete((res, t) ->
{
if (t != null)
@@ -90,24 +114,47 @@ public class InJVMTokenAwareVisitExecutor extends LoggingVisitor.LoggingVisitorE
}
}
+ protected NavigableMap<TokenPlacementModel.Range, List<TokenPlacementModel.Node>> getRing()
+ {
+ List<TokenPlacementModel.Node> other = peerStateToNodes(sut.cluster.coordinator(1).execute("select peer, tokens from system.peers", ConsistencyLevel.ONE));
+ List<TokenPlacementModel.Node> self = peerStateToNodes(sut.cluster.coordinator(1).execute("select broadcast_address, tokens from system.local", ConsistencyLevel.ONE));
+ List<TokenPlacementModel.Node> all = new ArrayList<>();
+ all.addAll(self);
+ all.addAll(other);
+ return TokenPlacementModel.replicate(all, rf);
+ }
+
+ protected Object[][] executeNodeLocal(String statement, TokenPlacementModel.Node node, Object... bindings)
+ {
+ IInstance instance = sut.cluster
+ .stream()
+ .filter((n) -> n.config().broadcastAddress().toString().contains(node.id))
+ .findFirst()
+ .get();
+ return instance.executeInternal(statement, bindings);
+ }
+
+
@JsonTypeName("in_jvm_token_aware")
public static class Configuration implements harry.core.Configuration.VisitorConfiguration
{
public final harry.core.Configuration.RowVisitorConfiguration row_visitor;
public final SystemUnderTest.ConsistencyLevel consistency_level;
-
+ public final int rf;
@JsonCreator
public Configuration(@JsonProperty("row_visitor") harry.core.Configuration.RowVisitorConfiguration rowVisitor,
- @JsonProperty("consistency_level") SystemUnderTest.ConsistencyLevel consistencyLevel)
+ @JsonProperty("consistency_level") SystemUnderTest.ConsistencyLevel consistencyLevel,
+ @JsonProperty("rf") int rf)
{
this.row_visitor = rowVisitor;
this.consistency_level = consistencyLevel;
+ this.rf = rf;
}
@Override
public Visitor make(Run run)
{
- return new GeneratingVisitor(run, new InJVMTokenAwareVisitExecutor(run, row_visitor, consistency_level));
+ return new GeneratingVisitor(run, new InJVMTokenAwareVisitExecutor(run, row_visitor, consistency_level, rf));
}
}
}
\ No newline at end of file
diff --git a/harry-integration/src/harry/runner/HarryRunnerJvm.java b/harry-core/src/harry/model/sut/injvm/InJvmSut.java
similarity index 61%
rename from harry-integration/src/harry/runner/HarryRunnerJvm.java
rename to harry-core/src/harry/model/sut/injvm/InJvmSut.java
index ed4ea46..46eaf9a 100644
--- a/harry-integration/src/harry/runner/HarryRunnerJvm.java
+++ b/harry-core/src/harry/model/sut/injvm/InJvmSut.java
@@ -16,28 +16,20 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.model.sut.injvm;
-import harry.core.Configuration;
-import harry.model.sut.InJvmSut;
+import org.apache.cassandra.distributed.api.ICluster;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
-import java.io.File;
-
-public class HarryRunnerJvm extends HarryRunner {
-
- public static void main(String[] args) throws Throwable {
- InJvmSut.init();
-
- HarryRunnerJvm runner = new HarryRunnerJvm();
- File configFile = runner.loadConfig(args);
-
- Configuration configuration = Configuration.fromFile(configFile);
- runner.run(configuration);
+public class InJvmSut extends InJvmSutBase<IInvokableInstance, ICluster<IInvokableInstance>>
+{
+ public InJvmSut(ICluster<IInvokableInstance> cluster)
+ {
+ super(cluster, 10);
}
-
- @Override
- public void beforeRun(Runner.TimedRunner runner) {
-
+ public InJvmSut(ICluster<IInvokableInstance> cluster, int threads)
+ {
+ super(cluster, threads);
}
}
diff --git a/harry-integration/src/harry/model/sut/InJvmSutBase.java b/harry-core/src/harry/model/sut/injvm/InJvmSutBase.java
similarity index 80%
rename from harry-integration/src/harry/model/sut/InJvmSutBase.java
rename to harry-core/src/harry/model/sut/injvm/InJvmSutBase.java
index aa2608f..500a472 100644
--- a/harry-integration/src/harry/model/sut/InJvmSutBase.java
+++ b/harry-core/src/harry/model/sut/injvm/InJvmSutBase.java
@@ -16,12 +16,14 @@
* limitations under the License.
*/
-package harry.model.sut;
+package harry.model.sut.injvm;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
-import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -36,21 +38,15 @@ import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import harry.core.Configuration;
-import org.apache.cassandra.distributed.api.Feature;
+import harry.model.sut.SystemUnderTest;
import org.apache.cassandra.distributed.api.ICluster;
import org.apache.cassandra.distributed.api.IInstance;
import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.distributed.api.IMessage;
import org.apache.cassandra.distributed.api.IMessageFilters;
-import relocated.shaded.com.google.common.collect.Iterators;
public class InJvmSutBase<NODE extends IInstance, CLUSTER extends ICluster<NODE>> implements SystemUnderTest.FaultInjectingSut
{
- public static void init()
- {
- Configuration.registerSubtypes(InJvmSutBaseConfiguration.class);
- }
-
private static final Logger logger = LoggerFactory.getLogger(InJvmSutBase.class);
// TODO: shut down properly
@@ -120,11 +116,11 @@ public class InJvmSutBase<NODE extends IInstance, CLUSTER extends ICluster<NODE>
}
else if (statement.contains("SELECT"))
{
- return Iterators.toArray(cluster
- // round-robin
- .coordinator(coordinator)
- .executeWithPaging(statement, toApiCl(cl), 1, bindings),
- Object[].class);
+ return toArray(cluster
+ // round-robin
+ .coordinator(coordinator)
+ .executeWithPaging(statement, toApiCl(cl), 1, bindings),
+ Object[].class);
}
else
{
@@ -139,10 +135,6 @@ public class InJvmSutBase<NODE extends IInstance, CLUSTER extends ICluster<NODE>
// TODO: find a better way to work around timeouts
if (t.getMessage().contains("timed out"))
return execute(statement, cl, coordinator, bindings);
-
- logger.error(String.format("Caught error while trying execute statement %s (%s): %s",
- statement, Arrays.toString(bindings), t.getMessage()),
- t);
throw t;
}
}
@@ -242,23 +234,7 @@ public class InJvmSutBase<NODE extends IInstance, CLUSTER extends ICluster<NODE>
CLUSTER cluster;
- cluster = cluster((cfg) -> {
- // TODO: make this configurable
- cfg.with(Feature.NETWORK, Feature.GOSSIP, Feature.NATIVE_PROTOCOL)
- .set("row_cache_size_in_mb", 10L)
- .set("index_summary_capacity_in_mb", 10L)
- .set("counter_cache_size_in_mb", 10L)
- .set("key_cache_size_in_mb", 10L)
- .set("file_cache_size_in_mb", 10)
- .set("memtable_heap_space_in_mb", 128)
- .set("memtable_offheap_space_in_mb", 128)
- .set("memtable_flush_writers", 1)
- .set("concurrent_compactors", 1)
- .set("concurrent_reads", 5)
- .set("concurrent_writes", 5)
- .set("compaction_throughput_mb_per_sec", 10)
- .set("hinted_handoff_enabled", false);
- },
+ cluster = cluster(defaultConfig(),
nodes,
new File(root));
@@ -267,6 +243,25 @@ public class InJvmSutBase<NODE extends IInstance, CLUSTER extends ICluster<NODE>
}
}
+ public static Consumer<IInstanceConfig> defaultConfig()
+ {
+ return (cfg) -> {
+ // TODO: make this configurable
+ cfg.set("row_cache_size", "256MiB")
+ .set("index_summary_capacity", "256MiB")
+ .set("counter_cache_size", "256MiB")
+ .set("key_cache_size", "256MiB")
+ .set("file_cache_size", "256MiB")
+ .set("memtable_heap_space", "512MiB")
+ .set("memtable_offheap_space", "512MiB")
+ .set("memtable_flush_writers", 2)
+ .set("concurrent_compactors", 2)
+ .set("concurrent_reads", 5)
+ .set("concurrent_writes", 5)
+ .set("compaction_throughput_mb_per_sec", 10)
+ .set("hinted_handoff_enabled", false);
+ };
+ }
public static org.apache.cassandra.distributed.api.ConsistencyLevel toApiCl(ConsistencyLevel cl)
{
switch (cl)
@@ -278,4 +273,13 @@ public class InJvmSutBase<NODE extends IInstance, CLUSTER extends ICluster<NODE>
}
throw new IllegalArgumentException("Don't know a CL: " + cl);
}
+
+ public static <T> T[] toArray(Iterator<? extends T> iterator, Class<T> klass) {
+ List<T> list = new ArrayList<>();
+ while (iterator.hasNext())
+ list.add(iterator.next());
+ T[] arr = (T[]) java.lang.reflect.Array.newInstance(klass, list.size());
+ return list.toArray(arr);
+ }
+
}
diff --git a/harry-core/src/harry/model/sut/injvm/MixedVersionInJvmSut.java b/harry-core/src/harry/model/sut/injvm/MixedVersionInJvmSut.java
new file mode 100644
index 0000000..33bf602
--- /dev/null
+++ b/harry-core/src/harry/model/sut/injvm/MixedVersionInJvmSut.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.model.sut.injvm;
+
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.distributed.api.ICluster;
+import org.apache.cassandra.distributed.api.IUpgradeableInstance;
+import org.apache.cassandra.distributed.shared.Versions;
+
+/**
+ * Make sure to set -Dcassandra.test.dtest_jar_path when using this class
+ */
+public class MixedVersionInJvmSut extends InJvmSutBase<IUpgradeableInstance, ICluster<IUpgradeableInstance>>
+{
+ private static final Logger logger = LoggerFactory.getLogger(MixedVersionInJvmSut.class);
+ private final Versions.Version initialVersion;
+ private final List<Versions.Version> versions;
+
+ public MixedVersionInJvmSut(ICluster<IUpgradeableInstance> cluster, Versions.Version initialVersion, List<Versions.Version> versions)
+ {
+ super(cluster, 10);
+ this.initialVersion = initialVersion;
+ this.versions = versions;
+ }
+
+
+ @Override
+ public void afterSchemaInit()
+ {
+ for (int i = 1; i <= cluster.size(); i++)
+ {
+ Versions.Version v = versions.get(i - 1);
+ if (!v.equals(initialVersion))
+ {
+ logger.info("Upgrading {} node from {} to {}", i, initialVersion, v);
+ IUpgradeableInstance instance = cluster.get(i);
+ try
+ {
+ instance.shutdown().get();
+ }
+ catch (Throwable e)
+ {
+ throw new RuntimeException(e);
+ }
+ instance.setVersion(v);
+ instance.startup();
+ }
+ else
+ {
+ logger.info("Skipping {} node upgrade, since it is already at the required version ({})", i, initialVersion);
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/harry-core/src/harry/model/sut/injvm/QuiescentLocalStateChecker.java b/harry-core/src/harry/model/sut/injvm/QuiescentLocalStateChecker.java
new file mode 100644
index 0000000..0deb2c8
--- /dev/null
+++ b/harry-core/src/harry/model/sut/injvm/QuiescentLocalStateChecker.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.model.sut.injvm;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NavigableMap;
+
+import harry.core.Run;
+import harry.model.QuiescentLocalStateCheckerBase;
+import harry.model.sut.TokenPlacementModel;
+import harry.util.ByteUtils;
+import harry.util.TokenUtil;
+import org.apache.cassandra.distributed.api.ConsistencyLevel;
+import org.apache.cassandra.distributed.api.IInstance;
+
+public class QuiescentLocalStateChecker extends QuiescentLocalStateCheckerBase
+{
+ public QuiescentLocalStateChecker(Run run)
+ {
+ super(run);
+ }
+
+ public static ModelFactory factory(int rf)
+ {
+ return (run) -> new QuiescentLocalStateChecker(run, rf);
+ }
+
+ public QuiescentLocalStateChecker(Run run, int rf)
+ {
+ super(run, rf);
+ }
+
+ @Override
+ protected NavigableMap<TokenPlacementModel.Range, List<TokenPlacementModel.Node>> getRing()
+ {
+ List<TokenPlacementModel.Node> other = TokenPlacementModel.peerStateToNodes(((InJvmSutBase<?, ?>) sut).cluster.coordinator(1).execute("select peer, tokens from system.peers", ConsistencyLevel.ONE));
+ List<TokenPlacementModel.Node> self = TokenPlacementModel.peerStateToNodes(((InJvmSutBase<?, ?>) sut).cluster.coordinator(1).execute("select broadcast_address, tokens from system.local", ConsistencyLevel.ONE));
+ List<TokenPlacementModel.Node> all = new ArrayList<>();
+ all.addAll(self);
+ all.addAll(other);
+ return TokenPlacementModel.replicate(all, rf);
+ }
+
+ @Override
+ protected Object[][] executeNodeLocal(String statement, TokenPlacementModel.Node node, Object... bindings)
+ {
+ IInstance instance = ((InJvmSutBase<?, ?>) sut).cluster
+ .stream()
+ .filter((n) -> n.config().broadcastAddress().toString().contains(node.id))
+ .findFirst()
+ .get();
+ return instance.executeInternal(statement, bindings);
+ }
+
+ protected long token(long pd)
+ {
+ return TokenUtil.token(ByteUtils.compose(ByteUtils.objectsToBytes(schema.inflatePartitionKey(pd))));
+ }
+}
diff --git a/harry-core/src/harry/reconciler/Reconciler.java b/harry-core/src/harry/reconciler/Reconciler.java
index 45649be..e4970dd 100644
--- a/harry-core/src/harry/reconciler/Reconciler.java
+++ b/harry-core/src/harry/reconciler/Reconciler.java
@@ -37,8 +37,10 @@ import harry.ddl.SchemaSpec;
import harry.model.OpSelectors;
import harry.operations.Query;
import harry.operations.QueryGenerator;
+import harry.runner.DataTracker;
import harry.util.BitSet;
import harry.util.Ranges;
+import harry.util.StringUtils;
import harry.visitors.GeneratingVisitor;
import harry.visitors.LtsVisitor;
import harry.visitors.ReplayingVisitor;
@@ -75,8 +77,7 @@ public class Reconciler
(processor) -> new GeneratingVisitor(run, processor));
}
- public Reconciler(Run run,
- Function<VisitExecutor, LtsVisitor> ltsVisitorFactory)
+ public Reconciler(Run run, Function<VisitExecutor, LtsVisitor> ltsVisitorFactory)
{
this.descriptorSelector = run.descriptorSelector;
this.pdSelector = run.pdSelector;
@@ -85,11 +86,11 @@ public class Reconciler
this.visitorFactory = ltsVisitorFactory;
}
- private final long debugCd = Long.getLong("harry.reconciler.debug_cd", -1L);
+ private final long debugCd = Long.getLong("debug_cd", -1L);
- public PartitionState inflatePartitionState(final long pd, long maxLts, Query query)
+ public PartitionState inflatePartitionState(final long pd, DataTracker tracker, Query query)
{
- PartitionState partitionState = new PartitionState();
+ PartitionState partitionState = new PartitionState(pd);
class Processor extends VisitExecutor
{
@@ -260,10 +261,12 @@ public class Reconciler
LtsVisitor visitor = visitorFactory.apply(new Processor());
long currentLts = pdSelector.minLtsFor(pd);
-
- while (currentLts <= maxLts && currentLts >= 0)
+ long maxStarted = tracker.maxStarted();
+ while (currentLts <= maxStarted && currentLts >= 0)
{
- visitor.visit(currentLts);
+ if (tracker.isFinished(currentLts))
+ visitor.visit(currentLts);
+
currentLts = pdSelector.nextLts(currentLts);
}
@@ -272,15 +275,18 @@ public class Reconciler
public class PartitionState implements Iterable<RowState>
{
+ private final long pd;
private final NavigableMap<Long, RowState> rows;
private RowState staticRow;
- private PartitionState()
+ private PartitionState(long pd)
{
+ this.pd = pd;
rows = new TreeMap<>();
if (!schema.staticColumns.isEmpty())
{
- staticRow = new RowState(STATIC_CLUSTERING,
+ staticRow = new RowState(this,
+ STATIC_CLUSTERING,
arr(schema.staticColumns.size(), NIL_DESCR),
arr(schema.staticColumns.size(), NO_TIMESTAMP));
}
@@ -308,9 +314,9 @@ public class Reconciler
return;
Iterator<Map.Entry<Long, RowState>> iter = rows.subMap(range.minBound, range.minInclusive,
- range.maxBound, range.maxInclusive)
- .entrySet()
- .iterator();
+ range.maxBound, range.maxInclusive)
+ .entrySet()
+ .iterator();
while (iter.hasNext())
{
Map.Entry<Long, RowState> e = iter.next();
@@ -357,7 +363,7 @@ public class Reconciler
}
}
- currentState = new RowState(cd, vdsCopy, ltss);
+ currentState = new RowState(this, cd, vdsCopy, ltss);
}
else
{
@@ -489,14 +495,17 @@ public class Reconciler
public static class RowState
{
public boolean hasPrimaryKeyLivenessInfo = false;
+ public final PartitionState partitionState;
public final long cd;
public final long[] vds;
public final long[] lts;
- public RowState(long cd,
+ public RowState(PartitionState partitionState,
+ long cd,
long[] vds,
long[] lts)
{
+ this.partitionState = partitionState;
this.cd = cd;
this.vds = vds;
this.lts = lts;
@@ -504,22 +513,21 @@ public class Reconciler
public String toString()
{
- return "RowState{" +
- "cd=" + (cd == STATIC_CLUSTERING ? "static" : cd) +
- ", vds=" + Arrays.toString(vds) +
- ", lts=" + Arrays.toString(lts) +
- '}';
+ return toString((SchemaSpec) null);
}
public String toString(SchemaSpec schema)
{
- return "RowState{" +
- "cd=" + (cd == STATIC_CLUSTERING ? "static" : cd) +
- ", vds=" + Arrays.toString(vds) +
- ", lts=" + Arrays.toString(lts) +
- ", clustering=" + (cd == STATIC_CLUSTERING ? "static" : Arrays.toString(schema.inflateClusteringKey(cd))) +
- ", values=" + Arrays.toString(cd == STATIC_CLUSTERING ? schema.inflateStaticColumns(vds) : schema.inflateRegularColumns(vds)) +
- '}';
+ return " rowStateRow("
+ + partitionState.pd +
+ "L, " + cd +
+ (partitionState.staticRow == null ? "" : ", values(" + StringUtils.toString(partitionState.staticRow.vds) + ")") +
+ (partitionState.staticRow == null ? "" : ", lts(" + StringUtils.toString(partitionState.staticRow.lts) + ")") +
+ ", values(" + StringUtils.toString(vds) + ")" +
+ ", lts(" + StringUtils.toString(lts) + ")" +
+ (schema == null ? "" : ", clustering=" + (cd == STATIC_CLUSTERING ? "static" : Arrays.toString(schema.inflateClusteringKey(cd)))) +
+ (schema == null ? "" : ", values=" + Arrays.toString(cd == STATIC_CLUSTERING ? schema.inflateStaticColumns(vds) : schema.inflateRegularColumns(vds))) +
+ ")";
}
}
}
\ No newline at end of file
diff --git a/harry-core/src/harry/runner/DataTracker.java b/harry-core/src/harry/runner/DataTracker.java
index 0171e43..82bd1dd 100644
--- a/harry-core/src/harry/runner/DataTracker.java
+++ b/harry-core/src/harry/runner/DataTracker.java
@@ -18,68 +18,49 @@
package harry.runner;
-import java.util.ArrayList;
-import java.util.List;
import java.util.function.LongConsumer;
import harry.core.Configuration;
+import harry.ddl.SchemaSpec;
+import harry.model.OpSelectors;
-public abstract class DataTracker
+public interface DataTracker
{
- protected List<LongConsumer> onStarted = new ArrayList<>();
- protected List<LongConsumer> onFinished = new ArrayList<>();
+ void onLtsStarted(LongConsumer onLts);
+ void onLtsFinished(LongConsumer onLts);
- public void onLtsStarted(LongConsumer onLts)
- {
- this.onStarted.add(onLts);
- }
-
- public void onLtsFinished(LongConsumer onLts)
- {
- this.onFinished.add(onLts);
- }
-
- public void started(long lts)
- {
- startedInternal(lts);
- for (LongConsumer consumer : onStarted)
- consumer.accept(lts);
- }
-
- public void finished(long lts)
- {
- finishedInternal(lts);
- for (LongConsumer consumer : onFinished)
- consumer.accept(lts);
- }
+ void beginModification(long lts);
+ void endModification(long lts);
- abstract void startedInternal(long lts);
- abstract void finishedInternal(long lts);
+ default void beginValidation(long pd) {}
+ default void endValidation(long pd) {}
- public abstract long maxStarted();
- public abstract long maxConsecutiveFinished();
+ long maxStarted();
+ boolean isFinished(long lts);
- public abstract Configuration.DataTrackerConfiguration toConfig();
+ Configuration.DataTrackerConfiguration toConfig();
- public static interface DataTrackerFactory
+ interface DataTrackerFactory
{
- DataTracker make();
+ DataTracker make(OpSelectors.PdSelector pdSelector, SchemaSpec schemaSpec);
}
- public static final DataTracker NO_OP = new NoOpDataTracker();
+ DataTracker NO_OP = new NoOpDataTracker();
- public static class NoOpDataTracker extends DataTracker
+ class NoOpDataTracker implements DataTracker
{
private NoOpDataTracker() {}
- protected void startedInternal(long lts) {}
- protected void finishedInternal(long lts) {}
+ public void onLtsStarted(LongConsumer onLts){}
+ public void onLtsFinished(LongConsumer onLts){}
+
+ public void beginModification(long lts){}
+ public void endModification(long lts){}
+
public long maxStarted() { return 0; }
- public long maxConsecutiveFinished() { return 0; }
+ public boolean isFinished(long lts) { return false; }
- public Configuration.DataTrackerConfiguration toConfig()
- {
- return null;
- }
+ public Configuration.DataTrackerConfiguration toConfig(){ return null; }
}
+
}
diff --git a/harry-core/src/harry/runner/DefaultDataTracker.java b/harry-core/src/harry/runner/DefaultDataTracker.java
index 85e2637..bf05cd9 100644
--- a/harry-core/src/harry/runner/DefaultDataTracker.java
+++ b/harry-core/src/harry/runner/DefaultDataTracker.java
@@ -19,42 +19,71 @@
package harry.runner;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.PriorityBlockingQueue;
import java.util.concurrent.atomic.AtomicLong;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.util.function.LongConsumer;
import harry.core.Configuration;
import harry.core.VisibleForTesting;
+import harry.concurrent.WaitQueue;
-public class DefaultDataTracker extends DataTracker
+public class DefaultDataTracker implements DataTracker
{
- private static final Logger logger = LoggerFactory.getLogger(DefaultDataTracker.class);
+ protected final AtomicLong maxSeenLts;
+ protected final AtomicLong maxCompleteLts;
+ protected final PriorityBlockingQueue<Long> reorderBuffer;
+ protected final DrainReorderQueueTask reorderTask;
- private final AtomicLong maxSeenLts;
- // TODO: This is a trivial implementation that can be significantly improved upon
- // for example, we could use a bitmap that records `1`s for all lts that are after
- // the consecutive, and "collapse" the bitmap state into the long as soon as we see
- // consecutive `1` on the left side.
- private final AtomicLong maxCompleteLts;
- private final PriorityBlockingQueue<Long> reorderBuffer;
+ protected List<LongConsumer> onStarted = new ArrayList<>();
+ protected List<LongConsumer> onFinished = new ArrayList<>();
public DefaultDataTracker()
{
this.maxSeenLts = new AtomicLong(-1);
this.maxCompleteLts = new AtomicLong(-1);
this.reorderBuffer = new PriorityBlockingQueue<>(100);
+ this.reorderTask = new DrainReorderQueueTask();
+ this.reorderTask.start();
+ }
+
+ @Override
+ public void onLtsStarted(LongConsumer onLts)
+ {
+ this.onStarted.add(onLts);
+ }
+
+ @Override
+ public void onLtsFinished(LongConsumer onLts)
+ {
+ this.onFinished.add(onLts);
}
- // TODO: there's also some room for improvement in terms of concurrency
- protected void startedInternal(long lts)
+ @Override
+ public void beginModification(long lts)
+ {
+ assert lts >= 0;
+ startedInternal(lts);
+ for (LongConsumer consumer : onStarted)
+ consumer.accept(lts);
+ }
+
+ @Override
+ public void endModification(long lts)
+ {
+ finishedInternal(lts);
+ for (LongConsumer consumer : onFinished)
+ consumer.accept(lts);
+ }
+
+ void startedInternal(long lts)
{
recordEvent(lts, false);
}
- protected void finishedInternal(long lts)
+ void finishedInternal(long lts)
{
recordEvent(lts, true);
}
@@ -67,44 +96,59 @@ public class DefaultDataTracker extends DataTracker
if (!finished)
return;
- long maxAchievedConsecutive = drainReorderQueue();
-
- if (maxAchievedConsecutive + 1 == lts)
- maxCompleteLts.compareAndSet(maxAchievedConsecutive, lts);
- else
+ if (!maxCompleteLts.compareAndSet(lts - 1, lts))
reorderBuffer.offer(lts);
+
+ reorderTask.notify.signalAll();
}
- public long drainReorderQueue()
+ private class DrainReorderQueueTask extends Thread
{
- final long expected = maxCompleteLts.get();
- long maxAchievedConsecutive = expected;
- if (reorderBuffer.isEmpty())
- return maxAchievedConsecutive;
-
- boolean catchingUp = false;
+ private final WaitQueue notify;
- Long smallest = reorderBuffer.poll();
- while (smallest != null && smallest == maxAchievedConsecutive + 1)
+ private DrainReorderQueueTask()
{
- maxAchievedConsecutive++;
- catchingUp = true;
- smallest = reorderBuffer.poll();
+ super("DrainReorderQueueTask");
+ this.notify = WaitQueue.newWaitQueue();
}
- // put back
- if (smallest != null)
- reorderBuffer.offer(smallest);
-
- if (catchingUp)
- maxCompleteLts.compareAndSet(expected, maxAchievedConsecutive);
+ public void run()
+ {
+ while (!Thread.interrupted())
+ {
+ try
+ {
+ WaitQueue.Signal signal = notify.register();
+ runOnce();
+ signal.awaitUninterruptibly();
+ }
+ catch (Throwable t)
+ {
+ t.printStackTrace();
+ }
+ }
+ }
- int bufferSize = reorderBuffer.size();
- if (bufferSize > 100)
- logger.warn("Reorder buffer size has grown up to " + reorderBuffer.size());
- return maxAchievedConsecutive;
+ public void runOnce()
+ {
+ long maxAchievedConsecutive = maxCompleteLts.get();
+ if (reorderBuffer.isEmpty())
+ return;
+
+ Long smallest = reorderBuffer.peek();
+ while (smallest != null && smallest == maxAchievedConsecutive + 1)
+ {
+ boolean res = maxCompleteLts.compareAndSet(maxAchievedConsecutive, smallest);
+ assert res : String.format("Should have exclusive access to maxCompleteLts, but someone wrote %d, while %d was expected", maxCompleteLts.get(), maxAchievedConsecutive);
+ maxAchievedConsecutive = smallest;
+ long removed = reorderBuffer.remove();
+ assert smallest == removed : String.format("Tried to remove %d but removed %d", smallest, removed);
+ smallest = reorderBuffer.peek();
+ }
+ }
}
+
public long maxStarted()
{
return maxSeenLts.get();
@@ -112,22 +156,30 @@ public class DefaultDataTracker extends DataTracker
public long maxConsecutiveFinished()
{
- if (!reorderBuffer.isEmpty())
- return drainReorderQueue();
-
return maxCompleteLts.get();
}
+ public boolean isFinished(long lts)
+ {
+ return lts <= maxConsecutiveFinished() || reorderBuffer.contains(lts);
+ }
+
public Configuration.DataTrackerConfiguration toConfig()
{
- return new Configuration.DefaultDataTrackerConfiguration(maxSeenLts.get(), maxCompleteLts.get());
+ return new Configuration.DefaultDataTrackerConfiguration(maxSeenLts.get(), maxCompleteLts.get(), new ArrayList<>(reorderBuffer));
}
@VisibleForTesting
- public void forceLts(long maxSeen, long maxComplete)
+ public void forceLts(long maxSeen, long maxComplete, List<Long> reorderBuffer)
{
+ System.out.printf("Forcing maxSeen: %d, maxComplete: %d, reorderBuffer: %s%n", maxSeen, maxComplete, reorderBuffer);
this.maxSeenLts.set(maxSeen);
this.maxCompleteLts.set(maxComplete);
+ if (reorderBuffer != null)
+ {
+ reorderBuffer.sort(Long::compareTo);
+ this.reorderBuffer.addAll(reorderBuffer);
+ }
}
public String toString()
diff --git a/harry-core/src/harry/runner/HarryRunner.java b/harry-core/src/harry/runner/HarryRunner.java
index 4dbfe76..32702bc 100644
--- a/harry-core/src/harry/runner/HarryRunner.java
+++ b/harry-core/src/harry/runner/HarryRunner.java
@@ -20,9 +20,6 @@ package harry.runner;
import java.io.File;
import java.io.FileNotFoundException;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,81 +32,37 @@ public abstract class HarryRunner
{
public static final Logger logger = LoggerFactory.getLogger(HarryRunner.class);
- protected final boolean localRun = Boolean.parseBoolean(System.getProperty("harry.local-run", "true"));
- protected final String cassandraVersion = System.getProperty("harry.cassandra-version");
- protected final long startedAt = Long.getLong("harry.start-time", System.currentTimeMillis());
-
- protected CompletableFuture<?> progress;
- protected ScheduledThreadPoolExecutor executor;
- public abstract void beforeRun(Runner.TimedRunner runner);
- public void afterRun(Runner runner, Object result)
- {
- boolean success = !(result instanceof Throwable);
-
- executor.shutdown();
- try
- {
- executor.awaitTermination(10, TimeUnit.SECONDS);
- }
- catch (InterruptedException e)
- {
- // ignore
- }
- }
-
public void run(Configuration config) throws Throwable
{
System.setProperty("cassandra.disable_tcactive_openssl", "true");
System.setProperty("relocated.shaded.io.netty.transport.noNative", "true");
System.setProperty("org.apache.cassandra.disable_mbean_registration", "true");
- executor = new ScheduledThreadPoolExecutor(1);
- executor.setRemoveOnCancelPolicy(true);
-
Runner runner = config.createRunner();
Run run = runner.getRun();
- progress = runner.initAndStartAll();
-
- assert runner instanceof Runner.TimedRunner : "Please use a timed runner at the top level.";
- beforeRun((Runner.TimedRunner) runner);
-
- Object result = null;
+ Throwable thrown = null;
try
{
- result = progress.handle((a, b) -> {
- if (b != null)
- return b;
- return a;
- }).get();
- if (result instanceof Throwable)
- logger.error("Execution failed!", (Throwable) result);
-
+ beforeRun(runner);
+ runner.run();
}
catch (Throwable e)
{
logger.error("Failed due to exception: " + e.getMessage(), e);
- result = e;
+ thrown = e;
}
finally
{
- afterRun(runner, result);
-
- logger.info("Shutting down executor..");
- tryRun(() -> {
- executor.shutdownNow();
- executor.awaitTermination(10, TimeUnit.SECONDS);
- });
-
logger.info("Shutting down runner..");
- tryRun(runner::shutdown);
- boolean failed = result instanceof Throwable;
+ boolean failed = thrown != null;
if (!failed)
{
logger.info("Shutting down cluster..");
tryRun(run.sut::shutdown);
}
+ afterRun(runner, thrown);
logger.info("Exiting...");
if (failed)
System.exit(1);
@@ -136,7 +89,7 @@ public abstract class HarryRunner
* @return Configuration YAML file.
* @throws Exception If file is not found or cannot be read.
*/
- public static File loadConfig(String[] args) throws Exception {
+ public static File loadConfig(String... args) throws Exception {
if (args == null || args.length == 0) {
throw new Exception("Harry config YAML not provided.");
}
@@ -152,4 +105,7 @@ public abstract class HarryRunner
return configFile;
}
+
+ public abstract void beforeRun(Runner runner);
+ public abstract void afterRun(Runner runner, Object result);
}
\ No newline at end of file
diff --git a/harry-core/src/harry/runner/LockingDataTracker.java b/harry-core/src/harry/runner/LockingDataTracker.java
new file mode 100644
index 0000000..a3d1968
--- /dev/null
+++ b/harry-core/src/harry/runner/LockingDataTracker.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.runner;
+
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
+
+import harry.concurrent.WaitQueue;
+import harry.core.Configuration;
+import harry.ddl.SchemaSpec;
+import harry.model.OpSelectors;
+
+/**
+ * Locking data tracker, that can be used with a quiescent model checker while providing
+ * a high degree of concurrency. It works by isolating readers from writers. In other words,
+ * readers can intersect with other readers, and writers can coincide with other writers.
+ *
+ * We achieve quiescence on a partition level, not on LTS level, and we know for sure
+ * which operations have finished for a partition, even if their LTS are non-contiguous.
+ *
+ * We use a simple wait queue for queuing up waiters, and a compact long counter for
+ * tracking the number of concurrent readers and writers. Lower 32 bits hold a number of
+ * readers, and higher 32 bits - a number of writers.
+ */
+public class LockingDataTracker extends DefaultDataTracker
+{
+ private final Map<Long, ReadersWritersLock> locked = new ConcurrentHashMap<>();
+
+ private final WaitQueue waitQueue = WaitQueue.newWaitQueue();
+ private final OpSelectors.PdSelector pdSelector;
+ private final SchemaSpec schemaSpec;
+
+ public LockingDataTracker(OpSelectors.PdSelector pdSelector, SchemaSpec schemaSpec)
+ {
+ this.pdSelector = pdSelector;
+ this.schemaSpec = schemaSpec;
+ }
+
+ @Override
+ public void beginModification(long lts)
+ {
+ ReadersWritersLock partitionLock = getLockForLts(lts);
+ partitionLock.lockForWrite();
+ super.beginModification(lts);
+ }
+
+ @Override
+ public void endModification(long lts)
+ {
+ super.endModification(lts);
+ getLockForLts(lts).unlockAfterWrite();
+ }
+
+ @Override
+ public void beginValidation(long pd)
+ {
+ ReadersWritersLock partitionLock = getLock(pd);
+ partitionLock.lockForRead();
+ super.beginValidation(pd);
+ }
+
+ @Override
+ public void endValidation(long pd)
+ {
+ super.endValidation(pd);
+ getLock(pd).unlockAfterRead();
+ }
+
+ public void validate(long pd, Runnable runnable)
+ {
+ ReadersWritersLock partitionLock = getLockForLts(pd);
+ partitionLock.lockForRead();
+ runnable.run();
+ partitionLock.unlockAfterRead();
+ }
+
+ private ReadersWritersLock getLockForLts(long lts)
+ {
+ long pd = pdSelector.pd(lts, schemaSpec);
+ return getLock(pd);
+ }
+
+ private ReadersWritersLock getLock(long pd)
+ {
+ return locked.computeIfAbsent(pd, (pd_) -> new ReadersWritersLock(waitQueue, pd));
+ }
+
+ /**
+ * Readers/writers lock. It was decided not to use signals here, and instead go for a
+ * busyspin instead, since we expect locks to be released briefly and contention to be minimal.
+ */
+ public static class ReadersWritersLock
+ {
+ private static final AtomicLongFieldUpdater<ReadersWritersLock> fieldUpdater = AtomicLongFieldUpdater.newUpdater(ReadersWritersLock.class, "lock");
+ private volatile long lock;
+
+ final long descriptor;
+ final WaitQueue waitQueue;
+
+ public ReadersWritersLock(WaitQueue waitQueue, long descriptor)
+ {
+ this.waitQueue = waitQueue;
+ this.lock = 0L;
+ this.descriptor = descriptor;
+ }
+
+ @Override
+ public String toString()
+ {
+ long lock = this.lock;
+ return "PartitionLock{" +
+ "pd = " + descriptor +
+ ", readers = " + getReaders(lock) +
+ ", writers = " + getWriters(lock) +
+ '}';
+ }
+
+ public void lockForWrite()
+ {
+ while (true)
+ {
+ WaitQueue.Signal signal = waitQueue.register();
+ long v = lock;
+ if (getReaders(v) == 0)
+ {
+ if (fieldUpdater.compareAndSet(this, v, incWriters(v)))
+ {
+ signal.cancel();
+ return;
+ }
+ }
+ signal.awaitUninterruptibly();
+ }
+ }
+
+ public void unlockAfterWrite()
+ {
+ while (true)
+ {
+ long v = lock;
+ if (fieldUpdater.compareAndSet(this, v, decWriters(v)))
+ {
+ waitQueue.signalAll();
+ return;
+ }
+ }
+ }
+
+ public void lockForRead()
+ {
+ while (true)
+ {
+ WaitQueue.Signal signal = waitQueue.register();
+ long v = lock;
+ if (getWriters(v) == 0)
+ {
+ if (fieldUpdater.compareAndSet(this, v, incReaders(v)))
+ {
+ signal.cancel();
+ return;
+ }
+ }
+ signal.awaitUninterruptibly();
+ }
+ }
+
+ public boolean tryLockForRead()
+ {
+ long v = lock;
+ if (getWriters(v) == 0 && fieldUpdater.compareAndSet(this, v, incReaders(v)))
+ return true;
+
+ return false;
+ }
+
+ public void unlockAfterRead()
+ {
+ while (true)
+ {
+ long v = lock;
+ if (fieldUpdater.compareAndSet(this, v, decReaders(v)))
+ {
+ waitQueue.signalAll();
+ return;
+ }
+ }
+ }
+
+ private long incReaders(long v)
+ {
+ long readers = getReaders(v);
+ v &= ~0xffffffffL; // erase all readers
+ return v | (readers + 1L);
+ }
+
+ private long decReaders(long v)
+ {
+ long readers = getReaders(v);
+ assert readers >= 1;
+ v &= ~0xffffffffL; // erase all readers
+ return v | (readers - 1L);
+ }
+
+ private long incWriters(long v)
+ {
+ long writers = getWriters(v);
+ v &= ~0xffffffff00000000L; // erase all writers
+ return v | ((writers + 1L) << 32);
+ }
+
+ private long decWriters(long v)
+ {
+ long writers = getWriters(v);
+ assert writers >= 1 : "Writers left " + writers;
+ v &= ~0xffffffff00000000L; // erase all writers
+ return v | ((writers - 1L) << 32);
+ }
+
+ public int getReaders(long v)
+ {
+ v &= 0xffffffffL;
+ return (int) v;
+ }
+
+ public int getWriters(long v)
+ {
+ v >>= 32;
+ v &= 0xffffffffL;
+ return (int) v;
+ }
+ }
+
+ @Override
+ public Configuration.DataTrackerConfiguration toConfig()
+ {
+ return new Configuration.LockingDataTrackerConfiguration(maxSeenLts.get(), maxCompleteLts.get(), new ArrayList<>(reorderBuffer));
+ }
+
+}
diff --git a/harry-core/src/harry/runner/Runner.java b/harry-core/src/harry/runner/Runner.java
index 207394d..1fc7592 100644
--- a/harry-core/src/harry/runner/Runner.java
+++ b/harry-core/src/harry/runner/Runner.java
@@ -24,57 +24,67 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
-import java.util.Collection;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
-import java.util.concurrent.CompletableFuture;
+import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.BooleanSupplier;
+import java.util.function.Consumer;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import harry.concurrent.ExecutorFactory;
+import harry.concurrent.Interruptible;
+import harry.concurrent.Shutdownable;
+import harry.concurrent.WaitQueue;
import harry.core.Configuration;
import harry.core.Run;
import harry.visitors.Visitor;
+import static harry.concurrent.Clock.Global.nanoTime;
+import static harry.concurrent.InfiniteLoopExecutor.Daemon.NON_DAEMON;
+import static harry.concurrent.InfiniteLoopExecutor.Interrupts.UNSYNCHRONIZED;
+import static harry.concurrent.InfiniteLoopExecutor.SimulatorSafe.SAFE;
+
public abstract class Runner
{
private static final Logger logger = LoggerFactory.getLogger(Runner.class);
protected final Run run;
protected final Configuration config;
- protected final ScheduledExecutorService executor;
// If there's an error, there's a good chance we're going to hit it more than once
// since we have multiple concurrent checkers running
- protected final CopyOnWriteArrayList<Throwable> errors;
+ protected final List<Throwable> errors;
- public Runner(Run run, Configuration config, int concurrency)
+ public Runner(Run run, Configuration config)
{
this.run = run;
this.config = config;
this.errors = new CopyOnWriteArrayList<>();
- this.executor = Executors.newScheduledThreadPool(concurrency);
}
- public Run getRun()
+ public void run() throws Throwable
{
- return run;
+ init(config, run);
+ runInternal();
+ teardown();
}
- public int getErrorCount()
+ protected abstract void runInternal() throws Throwable;
+
+ public Run getRun()
{
- return errors.size();
+ return run;
}
- public void init()
+ public static void init(Configuration config, Run run)
{
if (config.create_schema)
{
@@ -124,51 +134,8 @@ public abstract class Runner
}
}
- public CompletableFuture<?> initAndStartAll()
- {
- init();
- return start();
- }
-
public abstract String type();
- public abstract void shutdown() throws InterruptedException;
-
- protected CompletableFuture<?> start()
- {
- return start(true, () -> false);
- }
-
- protected abstract void shutDownVisitors();
-
- @SuppressWarnings("ResultOfMethodCallIgnored")
- protected void shutDownExecutors() throws InterruptedException
- {
- executor.shutdownNow();
- executor.awaitTermination(1, TimeUnit.MINUTES);
- }
-
- protected abstract CompletableFuture<?> start(boolean reportErrors, BooleanSupplier parentExit);
-
- protected Runnable reportThrowable(Runnable runnable, CompletableFuture<?> future)
- {
- return () ->
- {
- try
- {
- if (!future.isDone())
- runnable.run();
- }
- catch (Throwable t)
- {
- errors.add(t);
-
- if (!future.isDone())
- future.completeExceptionally(t);
- }
- };
- }
-
public interface RunnerFactory
{
Runner make(Run run, Configuration config);
@@ -178,39 +145,19 @@ public abstract class Runner
{
public final long runtime;
public final TimeUnit runtimeUnit;
-
- protected final ScheduledExecutorService shutdownExecutor;
- public TimedRunner(Run run, Configuration config, int concurrency, long runtime, TimeUnit runtimeUnit)
+ public TimedRunner(Run run, Configuration config, long runtime, TimeUnit runtimeUnit)
{
- super(run, config, concurrency);
+ super(run, config);
- this.shutdownExecutor = Executors.newSingleThreadScheduledExecutor();
this.runtime = runtime;
this.runtimeUnit = runtimeUnit;
}
+ }
- protected ScheduledFuture<?> scheduleTermination(AtomicBoolean terminated)
- {
- return shutdownExecutor.schedule(() ->
- {
- logger.info("Runner has reached configured runtime. Stopping...");
- // TODO: wait for the last full validation?
- terminated.set(true);
- },
- runtime,
- runtimeUnit);
- }
-
- @Override
- @SuppressWarnings("ResultOfMethodCallIgnored")
- protected void shutDownExecutors() throws InterruptedException
- {
- shutdownExecutor.shutdownNow();
- shutdownExecutor.awaitTermination(1, TimeUnit.MINUTES);
- executor.shutdownNow();
- executor.awaitTermination(1, TimeUnit.MINUTES);
- }
+ public static Configuration.RunnerConfiguration single(Visitor.VisitorFactory visitor)
+ {
+ return (run, config) -> new SingleVisitRunner(run, config, Collections.singletonList(visitor));
}
public static class SingleVisitRunner extends Runner
@@ -218,10 +165,10 @@ public abstract class Runner
private final List<Visitor> visitors;
public SingleVisitRunner(Run run,
- Configuration config,
- List<? extends Visitor.VisitorFactory> visitorFactories)
+ Configuration config,
+ List<? extends Visitor.VisitorFactory> visitorFactories)
{
- super(run, config, 1);
+ super(run, config);
this.visitors = visitorFactories.stream().map(factory -> factory.make(run)).collect(Collectors.toList());
}
@@ -231,50 +178,30 @@ public abstract class Runner
return "single";
}
- @Override
- protected CompletableFuture<?> start(boolean reportErrors, BooleanSupplier parentExit)
- {
- CompletableFuture<?> future = new CompletableFuture<>();
-
- if (reportErrors)
- future.whenComplete((a, b) -> maybeReportErrors());
-
- executor.submit(reportThrowable(() -> run(visitors, future, parentExit), future));
- return future;
- }
-
- private void run(List<Visitor> visitors, CompletableFuture<?> future, BooleanSupplier parentExit)
+ public void runInternal()
{
- for (Visitor value: visitors)
- {
- if (parentExit.getAsBoolean())
- break;
-
+ for (Visitor value : visitors)
value.visit();
- }
-
- future.complete(null);
}
+ }
- @Override
- public void shutdown() throws InterruptedException
- {
- logger.info("Shutting down...");
- shutDownVisitors();
-
- // we need to wait for all threads that use schema to stop before we can tear down and drop the table
- shutDownExecutors();
-
- teardown();
- }
+ public static Configuration.RunnerConfiguration sequential(Visitor.VisitorFactory visitor,
+ long runtime, TimeUnit runtimeUnit)
+ {
+ return (r, c) -> new SequentialRunner(r, c, Collections.singletonList(visitor), runtime, runtimeUnit);
+ }
- @Override
- protected void shutDownVisitors()
- {
- shutDownVisitors(visitors);
- }
+ public static Runner sequential(Run run,
+ Configuration config,
+ List<Visitor.VisitorFactory> poolConfigurations,
+ long runtime, TimeUnit runtimeUnit)
+ {
+ return new SequentialRunner(run, config, poolConfigurations, runtime, runtimeUnit);
}
+ /**
+ * Runs all visitors sequentially, in the loop, for a specified amount of time
+ */
public static class SequentialRunner extends TimedRunner
{
protected final List<Visitor> visitors;
@@ -284,7 +211,7 @@ public abstract class Runner
List<? extends Visitor.VisitorFactory> visitorFactories,
long runtime, TimeUnit runtimeUnit)
{
- super(run, config, 1, runtime, runtimeUnit);
+ super(run, config, runtime, runtimeUnit);
this.visitors = visitorFactories.stream().map(factory -> factory.make(run)).collect(Collectors.toList());
}
@@ -296,85 +223,38 @@ public abstract class Runner
}
@Override
- protected CompletableFuture<?> start(boolean reportErrors, BooleanSupplier parentExit)
+ public void runInternal() throws Throwable
{
- CompletableFuture<?> future = new CompletableFuture<>();
-
- if (reportErrors)
- future.whenComplete((a, b) -> maybeReportErrors());
-
- AtomicBoolean terminated = new AtomicBoolean(false);
- scheduleTermination(terminated);
- BooleanSupplier exit = () -> Thread.currentThread().isInterrupted() || future.isDone()
- || terminated.get() || parentExit.getAsBoolean();
-
- executor.submit(reportThrowable(() -> run(visitors, future, exit), future));
- return future;
- }
-
- protected void run(List<Visitor> visitors,
- CompletableFuture<?> future,
- BooleanSupplier exit)
- {
- while (!exit.getAsBoolean())
+ long deadline = nanoTime() + runtimeUnit.toNanos(runtime);;
+ while (nanoTime() < deadline)
{
- for (Visitor visitor: visitors)
- {
- if (exit.getAsBoolean())
- break;
+ for (Visitor visitor : visitors)
visitor.visit();
- }
}
-
- future.complete(null);
- }
-
- @Override
- public void shutdown() throws InterruptedException
- {
- logger.info("Shutting down...");
-
- shutDownVisitors();
-
- // we need to wait for all threads that use schema to stop before we can tear down and drop the table
- shutDownExecutors();
-
- teardown();
}
+ }
- @Override
- protected void shutDownVisitors()
- {
- shutDownVisitors(visitors);
- }
+ public static Runner concurrent(Configuration config,
+ List<Configuration.VisitorPoolConfiguration> poolConfigurations,
+ long runtime, TimeUnit runtimeUnit)
+ {
+ return new ConcurrentRunner(config.createRun(), config, poolConfigurations, runtime, runtimeUnit);
}
- // TODO: this requires some significant improvement
+ /**
+ * Runs all visitors concurrently, each visitor in its own thread, looped, for a specified amount of time
+ */
public static class ConcurrentRunner extends TimedRunner
{
- private final List<List<Visitor>> perThreadVisitors;
- private final int concurrency;
+ private final List<Configuration.VisitorPoolConfiguration> poolConfigurations;
public ConcurrentRunner(Run run,
Configuration config,
- int concurrency,
- List<? extends Visitor.VisitorFactory> visitorFactories,
+ List<Configuration.VisitorPoolConfiguration> poolConfigurations,
long runtime, TimeUnit runtimeUnit)
{
- super(run, config, concurrency, runtime, runtimeUnit);
-
- this.concurrency = concurrency;
- this.perThreadVisitors = new ArrayList<>(concurrency);
-
- for (int i = 0; i < concurrency; i++)
- {
- List<Visitor> visitors = new ArrayList<>();
-
- for (Visitor.VisitorFactory factory : visitorFactories)
- visitors.add(factory.make(run));
-
- perThreadVisitors.add(visitors);
- }
+ super(run, config, runtime, runtimeUnit);
+ this.poolConfigurations = poolConfigurations;
}
@Override
@@ -383,93 +263,122 @@ public abstract class Runner
return "concurrent";
}
- @Override
- protected CompletableFuture<?> start(boolean reportErrors, BooleanSupplier parentExit)
+ public void runInternal() throws Throwable
{
- CompletableFuture<?> future = new CompletableFuture<>();
-
- if (reportErrors)
- future.whenComplete((a, b) -> maybeReportErrors());
-
- AtomicBoolean terminated = new AtomicBoolean(false);
- scheduleTermination(terminated);
- BooleanSupplier exit = () -> Thread.currentThread().isInterrupted() || future.isDone()
- || terminated.get() || parentExit.getAsBoolean();
-
- AtomicInteger liveCount = new AtomicInteger(0);
-
- for (int i = 0; i < concurrency; i++)
- {
- List<Visitor> visitors = perThreadVisitors.get(i);
- executor.submit(reportThrowable(() -> run(visitors, future, exit, liveCount), future));
- }
-
- return future;
- }
+ List<Interruptible> threads = new ArrayList<>();
+ WaitQueue queue = WaitQueue.newWaitQueue();
+ WaitQueue.Signal interrupt = queue.register();
- private void run(List<Visitor> visitors,
- CompletableFuture<?> future,
- BooleanSupplier exit,
- AtomicInteger liveCount)
- {
- liveCount.incrementAndGet();
-
- while (!exit.getAsBoolean())
+ for (Configuration.VisitorPoolConfiguration poolConfiguration : poolConfigurations)
{
- for (Visitor visitor : visitors)
+ for (int i = 0; i < poolConfiguration.concurrency; i++)
{
- if (exit.getAsBoolean())
- break;
-
- visitor.visit();
+ Visitor visitor = poolConfiguration.visitor.make(run);
+ String name = String.format("%s-%d", poolConfiguration.prefix, i + 1);
+ Interruptible thread = ExecutorFactory.Global.executorFactory().infiniteLoop(name, wrapInterrupt((state) -> {
+ if (state == Interruptible.State.NORMAL)
+ visitor.visit();
+ }, interrupt::signal, errors::add), SAFE, NON_DAEMON, UNSYNCHRONIZED);
+ threads.add(thread);
}
}
-
- // If we're the last worker still running, complete the future....
- if (liveCount.decrementAndGet() == 0)
- future.complete(null);
+
+ interrupt.await(runtime, runtimeUnit);
+ shutdown(threads::stream);
+ if (!errors.isEmpty())
+ throw merge(errors);
}
+ }
- @Override
- public void shutdown() throws InterruptedException
- {
- logger.info("Shutting down...");
- shutDownVisitors();
+ public static Runner chain(Configuration config,
+ Configuration.RunnerConfiguration... runners)
+ {
+ return new ChainRunner(config.createRun(), config, Arrays.asList(runners));
+ }
- // we need to wait for all threads that use schema to stop before we can tear down and drop the table
- shutDownExecutors();
+ /**
+ * Chain runner is somewhat similar to StagedRunner in that it would execute multiple runners one after
+ * another, but unlike StagedRunner, it is single-shot: it will just run them once.
+ */
+ public static class ChainRunner extends Runner
+ {
+ private final List<Runner> stages;
- teardown();
+ public ChainRunner(Run run,
+ Configuration config,
+ List<Configuration.RunnerConfiguration> runnerFactories)
+ {
+ super(run, config);
+ this.stages = new ArrayList<>();
+ for (Configuration.RunnerConfiguration runner : runnerFactories)
+ stages.add(runner.make(run, config));
}
- @Override
- protected void shutDownVisitors()
+ protected void runInternal() throws Throwable
{
- shutDownVisitors(perThreadVisitors.stream().flatMap(Collection::stream).collect(Collectors.toList()));
+ for (Runner stage : stages)
+ stage.runInternal();
+ }
+
+ public String type()
+ {
+ return "chain";
}
}
- protected static void shutDownVisitors(List<Visitor> visitors)
+ public static void shutdown(Supplier<Stream<Interruptible>> threads)
{
- Throwable error = null;
-
- for (Visitor visitor : visitors)
- {
+ threads.get().forEach(Shutdownable::shutdown);
+ threads.get().forEach((interruptible) -> {
try
{
- visitor.shutdown();
+ if (!interruptible.awaitTermination(60, TimeUnit.SECONDS))
+ logger.info("Could not terminate before the timeout: " + threads.get().map(Shutdownable::isTerminated).collect(Collectors.toList()));
}
catch (InterruptedException e)
{
- if (error != null)
- error.addSuppressed(e);
- else
- error = e;
+ throw new RuntimeException(e);
}
+ });
+
+ }
+
+ public static Interruptible.Task wrapInterrupt(Interruptible.Task runnable, Runnable signalStop, Consumer<Throwable> registerException)
+ {
+ return (state) -> {
+ try
+ {
+ runnable.run(state);
+ }
+ catch (InterruptedException t)
+ {
+ signalStop.run();
+ }
+ catch (Throwable t)
+ {
+ // Since some of the exceptions are thrown from inside instances
+ if (!t.getClass().toString().contains("InterruptedException"))
+ registerException.accept(t);
+ signalStop.run();
+ }
+ };
+ }
+
+ public static Throwable merge(List<Throwable> existingFail)
+ {
+ if (existingFail.size() == 1)
+ return existingFail.get(0);
+
+ Throwable e = existingFail.get(0);
+ Throwable ret = e;
+ for (int i = 1; i < existingFail.size(); i++)
+ {
+ Throwable current = existingFail.get(i);
+ e.addSuppressed(current);
+ e = current;
}
-
- if (error != null)
- logger.warn("Failed to shut down all visitors!", error);
+
+ return ret;
}
private static void dumpExceptionToFile(BufferedWriter bw, Throwable throwable) throws IOException
@@ -496,7 +405,7 @@ public abstract class Runner
bw.newLine();
}
- private static void dumpStateToFile(Run run, Configuration config, List<Throwable> t)
+ public static void dumpStateToFile(Run run, Configuration config, List<Throwable> t)
{
try
{
@@ -518,12 +427,9 @@ public abstract class Runner
builder.setClock(run.clock.toConfig());
builder.setDataTracker(run.tracker.toConfig());
- try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file))))
- {
- bw.write(Configuration.toYamlString(builder.build()));
- bw.flush();
- }
+ Configuration.toFile(file, builder.build());
}
+
catch (Throwable e)
{
logger.error("Caught an error while trying to dump to file", e);
diff --git a/harry-core/src/harry/runner/StagedRunner.java b/harry-core/src/harry/runner/StagedRunner.java
index 88ebcdb..4e61f1e 100644
--- a/harry-core/src/harry/runner/StagedRunner.java
+++ b/harry-core/src/harry/runner/StagedRunner.java
@@ -18,34 +18,22 @@
package harry.runner;
+import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.BooleanSupplier;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import harry.core.Configuration;
import harry.core.Run;
+import static harry.concurrent.Clock.Global.nanoTime;
+
public class StagedRunner extends Runner.TimedRunner
{
public static final String TYPE = "staged";
- public static void register()
- {
- Configuration.registerSubtypes(StagedRunner.StagedRunnerConfig.class);
- }
-
- private static final Logger logger = LoggerFactory.getLogger(StagedRunner.class);
-
- private final List<Configuration.RunnerConfiguration> runnerFactories;
private final List<Runner> stages;
public StagedRunner(Run run,
@@ -53,9 +41,10 @@ public class StagedRunner extends Runner.TimedRunner
List<Configuration.RunnerConfiguration> runnerFactories,
long runtime, TimeUnit runtimeUnit)
{
- super(run, config, 1, runtime, runtimeUnit);
- this.runnerFactories = runnerFactories;
- this.stages = new CopyOnWriteArrayList<>();
+ super(run, config, runtime, runtimeUnit);
+ this.stages = new ArrayList<>();
+ for (Configuration.RunnerConfiguration runner : runnerFactories)
+ stages.add(runner.make(run, config));
}
@Override
@@ -64,112 +53,14 @@ public class StagedRunner extends Runner.TimedRunner
}
@Override
- protected CompletableFuture<?> start(boolean reportErrors, BooleanSupplier parentExit)
+ protected void runInternal() throws Throwable
{
- CompletableFuture<?> future = new CompletableFuture<>();
-
- if (reportErrors)
- {
- future.whenComplete((a, b) ->
- {
- collectErrors();
- maybeReportErrors();
- });
- }
-
- AtomicBoolean terminated = new AtomicBoolean(false);
- scheduleTermination(terminated);
-
- BooleanSupplier exit = () -> Thread.currentThread().isInterrupted() || future.isDone()
- || terminated.get() || parentExit.getAsBoolean();
-
- executor.submit(() ->
- {
- for (Configuration.RunnerConfiguration runnerConfig : runnerFactories)
- {
- try
- {
- Runner runner = runnerConfig.make(run, config);
- if (runner instanceof StagedRunner)
- throw new IllegalArgumentException("StagedRunner can not be nested inside of a StagedRunner");
- stages.add(runner);
- }
- catch (Throwable t)
- {
- future.completeExceptionally(t);
- }
- }
-
- while (!exit.getAsBoolean())
- {
- logger.info("Starting next staged run...");
- int stage = 1;
-
- for (Runner runner : stages)
- {
- if (exit.getAsBoolean())
- break;
-
- try
- {
- logger.info("Starting stage {}: {}...", stage++, runner.type());
- runner.start(false, exit).get();
-
- if (!terminated.get())
- logger.info("...stage complete.");
-
- // Wait for any previous runners to settle down...
- while (run.tracker.maxConsecutiveFinished() != run.tracker.maxStarted())
- {
- TimeUnit.SECONDS.sleep(1);
- logger.warn("Waiting for any previous runners to settle down: {}",
- run.tracker.toString());
- }
-
- }
- catch (Throwable t)
- {
- future.completeExceptionally(t);
- break;
- }
- }
-
- if (!terminated.get())
- logger.info("All stages complete!");
- }
+ long deadline = nanoTime() + runtimeUnit.toNanos(nanoTime());
- future.complete(null);
- });
-
- return future;
- }
-
- @Override
- public void shutdown() throws InterruptedException
- {
- logger.info("Shutting down...");
-
- for (Runner runner : stages)
- {
- runner.shutDownVisitors();
- runner.shutDownExecutors();
- }
-
- shutDownExecutors();
- teardown();
- }
-
- @Override
- protected void shutDownVisitors()
- {
- // Visitors are shut down in the sub-runners.
- }
-
- private void collectErrors()
- {
- for (Runner runner : stages)
+ while (nanoTime() < deadline)
{
- errors.addAll(runner.errors);
+ for (Runner runner : stages)
+ runner.runInternal();
}
}
diff --git a/harry-integration/src/harry/runner/TrivialShrinker.java b/harry-core/src/harry/runner/TrivialShrinker.java
similarity index 98%
rename from harry-integration/src/harry/runner/TrivialShrinker.java
rename to harry-core/src/harry/runner/TrivialShrinker.java
index b598b85..d82c0b6 100644
--- a/harry-integration/src/harry/runner/TrivialShrinker.java
+++ b/harry-core/src/harry/runner/TrivialShrinker.java
@@ -92,7 +92,7 @@ public class TrivialShrinker
if (pdsToSkip.contains(pdToCheck))
continue;
pdsToSkip.add(pdToCheck);
- config.make(run, configuration).init();
+ Runner.init(configuration, run);
try
{
@@ -125,7 +125,7 @@ public class TrivialShrinker
if (ltsToSkip.contains(ltsToCheck) || pdsToSkip.contains(run.pdSelector.pd(lts, run.schemaSpec)))
continue;
ltsToSkip.add(ltsToCheck);
- config.make(run, configuration).init();
+ Runner.init(configuration, run);
try
{
diff --git a/harry-core/src/harry/runner/UpToLtsRunner.java b/harry-core/src/harry/runner/UpToLtsRunner.java
index f89ecbb..f34df45 100644
--- a/harry-core/src/harry/runner/UpToLtsRunner.java
+++ b/harry-core/src/harry/runner/UpToLtsRunner.java
@@ -18,11 +18,9 @@
package harry.runner;
+import java.util.Collections;
import java.util.List;
-import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.BooleanSupplier;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -31,16 +29,31 @@ import harry.core.Configuration;
import harry.core.Run;
import harry.visitors.Visitor;
+import static harry.concurrent.Clock.Global.nanoTime;
+
+/**
+ * Runner that allows to run for a specific number of logical timestamps, rather
+ * than being configured by time.
+ */
public class UpToLtsRunner extends Runner.SequentialRunner
{
public static final String TYPE = "up_to_lts";
- public static void register()
+ private final long maxLts;
+
+ public static Configuration.RunnerConfiguration factory(List<? extends Visitor.VisitorFactory> visitorFactories,
+ long maxLts,
+ long runtime, TimeUnit runtimeUnit)
{
- Configuration.registerSubtypes(UpToLtsRunnerConfig.class);
+ return (r, c) -> new UpToLtsRunner(r, c, visitorFactories, maxLts, runtime, runtimeUnit);
}
- private final long maxLts;
+ public static Configuration.RunnerConfiguration factory(Visitor.VisitorFactory visitorFactory,
+ long maxLts,
+ long runtime, TimeUnit runtimeUnit)
+ {
+ return (r, c) -> new UpToLtsRunner(r, c, Collections.singletonList(visitorFactory), maxLts, runtime, runtimeUnit);
+ }
public UpToLtsRunner(Run run,
Configuration config,
@@ -59,21 +72,14 @@ public class UpToLtsRunner extends Runner.SequentialRunner
}
@Override
- protected CompletableFuture<?> start(boolean reportErrors, BooleanSupplier parentExit)
+ public void runInternal()
{
- CompletableFuture<?> future = new CompletableFuture<>();
-
- if (reportErrors)
- future.whenComplete((a, b) -> maybeReportErrors());
-
- AtomicBoolean terminated = new AtomicBoolean(false);
- scheduleTermination(terminated);
- BooleanSupplier exit = () -> run.clock.peek() >= maxLts
- || Thread.currentThread().isInterrupted() || future.isDone()
- || terminated.get() || parentExit.getAsBoolean();
-
- executor.submit(reportThrowable(() -> run(visitors, future, exit), future));
- return future;
+ long deadline = nanoTime() + runtimeUnit.toNanos(runtime);
+ while (run.tracker.maxStarted() < maxLts && nanoTime() < deadline)
+ {
+ for (Visitor visitor : visitors)
+ visitor.visit();
+ }
}
@JsonTypeName(TYPE)
diff --git a/harry-core/src/harry/util/ByteUtils.java b/harry-core/src/harry/util/ByteUtils.java
index 4adf536..17153a3 100644
--- a/harry-core/src/harry/util/ByteUtils.java
+++ b/harry-core/src/harry/util/ByteUtils.java
@@ -92,6 +92,13 @@ public class ByteUtils
return b;
}
+ public static ByteBuffer[] objectsToBytes(Object... objects)
+ {
+ ByteBuffer[] bytes = new ByteBuffer[objects.length];
+ for (int i = 0; i < objects.length; i++)
+ bytes[i] = objectToBytes(objects[i]);
+ return bytes;
+ }
public static ByteBuffer objectToBytes(Object obj)
{
@@ -131,46 +138,6 @@ public class ByteUtils
obj.getClass()));
}
- public static ByteBuffer pack(Collection<ByteBuffer> buffers, int elements)
- {
- int size = 0;
- for (ByteBuffer bb : buffers)
- size += sizeOfValue(bb);
-
- ByteBuffer result = ByteBuffer.allocate(sizeOfCollectionSize(elements) + size);
- writeCollectionSize(result, elements);
- for (ByteBuffer bb : buffers)
- writeValue(result, bb);
- result.flip();
- return result;
- }
-
- public static int sizeOfValue(ByteBuffer value)
- {
- return value == null ? 4 : 4 + value.remaining();
- }
-
- protected static void writeCollectionSize(ByteBuffer output, int elements)
- {
- output.putInt(elements);
- }
- public static void writeValue(ByteBuffer output, ByteBuffer value)
- {
- if (value == null)
- {
- output.putInt(-1);
- return;
- }
-
- output.putInt(value.remaining());
- output.put(value.duplicate());
- }
-
- protected static int sizeOfCollectionSize(int elements)
- {
- return 4;
- }
-
public static ByteBuffer compose(ByteBuffer... buffers) {
if (buffers.length == 1) return buffers[0];
diff --git a/harry-integration/test/harry/model/TestBaseImpl.java b/harry-core/src/harry/util/StringUtils.java
similarity index 72%
copy from harry-integration/test/harry/model/TestBaseImpl.java
copy to harry-core/src/harry/util/StringUtils.java
index 2a91086..45a0b6a 100644
--- a/harry-integration/test/harry/model/TestBaseImpl.java
+++ b/harry-core/src/harry/util/StringUtils.java
@@ -16,17 +16,20 @@
* limitations under the License.
*/
-package harry.model;
+package harry.util;
-import org.junit.BeforeClass;
-
-import org.apache.cassandra.distributed.api.ICluster;
-
-// TODO: probably redundant
-public class TestBaseImpl extends org.apache.cassandra.distributed.test.TestBaseImpl
+public class StringUtils
{
- @BeforeClass
- public static void beforeClass() throws Throwable {
- ICluster.setup();
+ public static String toString(long[] arr)
+ {
+ String s = "";
+ for (int i = 0; i < arr.length; i++)
+ {
+ s += arr[i];
+ s += "L";
+ if (i < (arr.length - 1))
+ s += ',';
+ }
+ return s;
}
}
diff --git a/harry-core/src/harry/visitors/AllPartitionsValidator.java b/harry-core/src/harry/visitors/AllPartitionsValidator.java
index 7bd5b50..be37d34 100644
--- a/harry-core/src/harry/visitors/AllPartitionsValidator.java
+++ b/harry-core/src/harry/visitors/AllPartitionsValidator.java
@@ -18,16 +18,19 @@
package harry.visitors;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.LongPredicate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import harry.concurrent.ExecutorFactory;
+import harry.concurrent.Interruptible;
+import harry.concurrent.WaitQueue;
+import harry.core.Configuration;
import harry.core.MetricReporter;
import harry.core.Run;
import harry.ddl.SchemaSpec;
@@ -35,9 +38,16 @@ import harry.model.Model;
import harry.model.OpSelectors;
import harry.model.sut.SystemUnderTest;
import harry.operations.Query;
+import harry.runner.DataTracker;
+import harry.runner.Runner;
-// This might be something that potentially grows into the validator described in the design doc;
-// right now it's just a helper/container class
+import static harry.concurrent.InfiniteLoopExecutor.Daemon.NON_DAEMON;
+import static harry.concurrent.InfiniteLoopExecutor.Interrupts.UNSYNCHRONIZED;
+import static harry.concurrent.InfiniteLoopExecutor.SimulatorSafe.SAFE;
+
+/**
+ * Concurrently validates all partitions that were visited during this run.
+ */
public class AllPartitionsValidator implements Visitor
{
private static final Logger logger = LoggerFactory.getLogger(AllPartitionsValidator.class);
@@ -48,21 +58,32 @@ public class AllPartitionsValidator implements Visitor
protected final OpSelectors.MonotonicClock clock;
protected final OpSelectors.PdSelector pdSelector;
protected final MetricReporter metricReporter;
- protected final ExecutorService executor;
protected final SystemUnderTest sut;
+ protected final DataTracker tracker;
- protected final AtomicBoolean condition;
- protected final AtomicLong maxPos;
-
+ protected final LongPredicate condition;
protected final int concurrency;
- protected final int triggerAfter;
- public AllPartitionsValidator(int concurrency,
- int triggerAfter,
- Run run,
+ public static Configuration.VisitorConfiguration factory(int concurrency,
+ LongPredicate condition,
+ Model.ModelFactory modelFactory)
+ {
+ return (r) -> new AllPartitionsValidator(r, concurrency, condition, modelFactory);
+ }
+
+ public AllPartitionsValidator(Run run,
+ int concurrency,
+ long triggerAfter,
+ Model.ModelFactory modelFactory)
+ {
+ this(run, concurrency, (lts) -> (lts > 0 && lts % triggerAfter == 0), modelFactory);
+ }
+
+ public AllPartitionsValidator(Run run,
+ int concurrency,
+ LongPredicate condition,
Model.ModelFactory modelFactory)
{
- this.triggerAfter = triggerAfter;
this.metricReporter = run.metricReporter;
this.model = modelFactory.make(run);
this.schema = run.schemaSpec;
@@ -70,68 +91,57 @@ public class AllPartitionsValidator implements Visitor
this.sut = run.sut;
this.pdSelector = run.pdSelector;
this.concurrency = concurrency;
- this.executor = Executors.newFixedThreadPool(concurrency);
- this.condition = new AtomicBoolean();
- this.maxPos = new AtomicLong(-1);
- run.tracker.onLtsStarted((lts) -> {
- maxPos.updateAndGet(current -> Math.max(pdSelector.positionFor(lts), current));
- if (triggerAfter == 0 || (triggerAfter > 0 && lts % triggerAfter == 0))
- condition.set(true);
- });
+ this.tracker = run.tracker;
+ this.condition = condition;
}
- protected CompletableFuture<Void> validateAllPartitions(ExecutorService executor, int parallelism)
+ protected void validateAllPartitions() throws Throwable
{
- final long maxPos = this.maxPos.get();
- AtomicLong cnt = new AtomicLong();
- CompletableFuture<?>[] futures = new CompletableFuture[parallelism];
- AtomicBoolean isDone = new AtomicBoolean(false);
-
- for (int i = 0; i < parallelism; i++)
+ List<Interruptible> threads = new ArrayList<>();
+ WaitQueue queue = WaitQueue.newWaitQueue();
+ WaitQueue.Signal interrupt = queue.register();
+ List<Throwable> errors = new CopyOnWriteArrayList<>();
+
+ final long maxPosition = pdSelector.maxPosition(tracker.maxStarted());
+ AtomicLong currentPosition = new AtomicLong();
+ for (int i = 0; i < concurrency; i++)
{
- futures[i] = CompletableFuture.supplyAsync(() -> {
- long pos;
- while ((pos = cnt.getAndIncrement()) < maxPos && !executor.isShutdown() && !Thread.interrupted() && !isDone.get())
- {
- if (pos > 0 && pos % 100 == 0)
- logger.info(String.format("Validated %d out of %d partitions", pos, maxPos));
- long visitLts = pdSelector.minLtsAt(pos);
-
- metricReporter.validatePartition();
-
- for (boolean reverse : new boolean[]{ true, false })
- {
- try
- {
- model.validate(Query.selectPartition(schema, pdSelector.pd(visitLts, schema), reverse));
- }
- catch (Throwable t)
- {
- isDone.set(true);
- logger.error("Caught an error while validating all partitions.", t);
- throw t;
- }
- }
- }
- return null;
- }, executor);
+ Interruptible thread = ExecutorFactory.Global.executorFactory().infiniteLoop(String.format("AllPartitionsValidator-%d", i + 1),
+ Runner.wrapInterrupt((state) -> {
+ if (state == Interruptible.State.NORMAL)
+ {
+ metricReporter.validatePartition();
+ long pos = currentPosition.getAndIncrement();
+ if (pos < maxPosition)
+ {
+ for (boolean reverse : new boolean[]{ true, false })
+ model.validate(Query.selectPartition(schema, pdSelector.pd(pdSelector.minLtsAt(pos), schema), reverse));
+ }
+ else
+ {
+ interrupt.signalAll();
+ }
+ }
+ }, interrupt::signal, errors::add), SAFE, NON_DAEMON, UNSYNCHRONIZED);
+ threads.add(thread);
}
- return CompletableFuture.allOf(futures);
+ interrupt.awaitUninterruptibly();
+
+ Runner.shutdown(threads::stream);
+ if (!errors.isEmpty())
+ throw Runner.merge(errors);
}
public void visit()
{
- // TODO: this is ok for now, but if/when we bring exhaustive checker back, we need to change this:
- // we'll probably want a low-concurrency validator somewhere in background all the time.
- if (condition.compareAndSet(true, false))
+ long lts = clock.peek();
+ if (condition.test(lts))
{
- long lts = clock.peek();
logger.info("Starting validation of all partitions as of lts {}...", lts);
-
try
{
- validateAllPartitions(executor, concurrency).get();
+ validateAllPartitions();
}
catch (Throwable e)
{
@@ -142,11 +152,5 @@ public class AllPartitionsValidator implements Visitor
}
}
- @SuppressWarnings("ResultOfMethodCallIgnored")
- @Override
- public void shutdown() throws InterruptedException
- {
- executor.shutdown();
- executor.awaitTermination(60, TimeUnit.SECONDS);
- }
+
}
\ No newline at end of file
diff --git a/harry-core/src/harry/visitors/CorruptingVisitor.java b/harry-core/src/harry/visitors/CorruptingVisitor.java
index 4677ea2..e8a028e 100644
--- a/harry-core/src/harry/visitors/CorruptingVisitor.java
+++ b/harry-core/src/harry/visitors/CorruptingVisitor.java
@@ -52,6 +52,7 @@ public class CorruptingVisitor implements Visitor
HideRowCorruptor::new),
new AddExtraRowCorruptor(run.schemaSpec,
run.clock,
+ run.tracker,
run.descriptorSelector),
new QueryResponseCorruptor.SimpleQueryResponseCorruptor(run.schemaSpec,
run.clock,
diff --git a/harry-integration/src/harry/runner/FaultInjectingVisitor.java b/harry-core/src/harry/visitors/FaultInjectingVisitor.java
similarity index 93%
rename from harry-integration/src/harry/runner/FaultInjectingVisitor.java
rename to harry-core/src/harry/visitors/FaultInjectingVisitor.java
index 4cca1b2..3885c4e 100644
--- a/harry-integration/src/harry/runner/FaultInjectingVisitor.java
+++ b/harry-core/src/harry/visitors/FaultInjectingVisitor.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executors;
@@ -31,33 +31,14 @@ import harry.core.Configuration;
import harry.core.Run;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
-import harry.visitors.LoggingVisitor;
-import harry.visitors.OperationExecutor;
-import harry.visitors.Visitor;
+/**
+ * Fault injecting visitor: randomly fails some of the queries.
+ *
+ * Requires {@code FaultInjectingSut} to function.
+ */
public class FaultInjectingVisitor extends LoggingVisitor
{
- public static void init()
- {
- Configuration.registerSubtypes(FaultInjectingVisitorConfiguration.class);
- }
-
- @JsonTypeName("fault_injecting")
- public static class FaultInjectingVisitorConfiguration extends Configuration.MutatingVisitorConfiguation
- {
- @JsonCreator
- public FaultInjectingVisitorConfiguration(@JsonProperty("row_visitor") Configuration.RowVisitorConfiguration row_visitor)
- {
- super(row_visitor);
- }
-
- @Override
- public Visitor make(Run run)
- {
- return new FaultInjectingVisitor(run, row_visitor);
- }
- }
-
private final AtomicInteger cnt = new AtomicInteger();
private final SystemUnderTest.FaultInjectingSut sut;
@@ -96,4 +77,21 @@ public class FaultInjectingVisitor extends LoggingVisitor
originator.complete(res);
});
}
+
+ @JsonTypeName("fault_injecting")
+ public static class FaultInjectingVisitorConfiguration extends Configuration.MutatingVisitorConfiguation
+ {
+ @JsonCreator
+ public FaultInjectingVisitorConfiguration(@JsonProperty("row_visitor") Configuration.RowVisitorConfiguration row_visitor)
+ {
+ super(row_visitor);
+ }
+
+ @Override
+ public Visitor make(Run run)
+ {
+ return new FaultInjectingVisitor(run, row_visitor);
+ }
+ }
+
}
diff --git a/harry-core/src/harry/visitors/MutatingVisitor.java b/harry-core/src/harry/visitors/MutatingVisitor.java
index db27216..c8f45f7 100644
--- a/harry-core/src/harry/visitors/MutatingVisitor.java
+++ b/harry-core/src/harry/visitors/MutatingVisitor.java
@@ -25,10 +25,12 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import harry.core.Configuration;
import harry.core.Run;
import harry.model.OpSelectors;
import harry.model.sut.SystemUnderTest;
@@ -39,12 +41,27 @@ public class MutatingVisitor extends GeneratingVisitor
{
private static final Logger logger = LoggerFactory.getLogger(MutatingVisitor.class);
+ public MutatingVisitor(Run run)
+ {
+ this(run, MutatingRowVisitor::new);
+ }
+
public MutatingVisitor(Run run,
OperationExecutor.RowVisitorFactory rowVisitorFactory)
{
this(run, new MutatingVisitExecutor(run, rowVisitorFactory.make(run)));
}
+ public static Configuration.VisitorConfiguration factory()
+ {
+ return MutatingVisitor::new;
+ }
+
+ public static Configuration.VisitorConfiguration factory(Function<Run, VisitExecutor> rowVisitorFactory)
+ {
+ return (r) -> new MutatingVisitor(r, rowVisitorFactory.apply(r));
+ }
+
public MutatingVisitor(Run run,
VisitExecutor visitExecutor)
{
@@ -64,25 +81,33 @@ public class MutatingVisitor extends GeneratingVisitor
protected final DataTracker tracker;
protected final SystemUnderTest sut;
protected final OperationExecutor rowVisitor;
+ protected final SystemUnderTest.ConsistencyLevel consistencyLevel;
private final int maxRetries = 10;
public MutatingVisitExecutor(Run run, OperationExecutor rowVisitor)
+ {
+ this(run, rowVisitor, SystemUnderTest.ConsistencyLevel.QUORUM);
+ }
+
+ public MutatingVisitExecutor(Run run, OperationExecutor rowVisitor, SystemUnderTest.ConsistencyLevel consistencyLevel)
{
this.descriptorSelector = run.descriptorSelector;
this.tracker = run.tracker;
this.sut = run.sut;
this.rowVisitor = rowVisitor;
+ this.consistencyLevel = consistencyLevel;
}
@Override
public void beforeLts(long lts, long pd)
{
- tracker.started(lts);
+ tracker.beginModification(lts);
}
@Override
public void afterLts(long lts, long pd)
{
+ // TODO: switch to Cassandra futures!
for (CompletableFuture<?> future : futures)
{
try
@@ -91,11 +116,15 @@ public class MutatingVisitor extends GeneratingVisitor
}
catch (Throwable t)
{
- throw new IllegalStateException("Couldn't repeat operations within timeout bounds.", t);
+ int complete = 0;
+ for (CompletableFuture<?> f : futures)
+ if (f.isDone()) complete++;
+
+ throw new IllegalStateException(String.format("Couldn't repeat operations within timeout bounds. %d out of %d futures complete", complete, futures.size()), t);
}
}
futures.clear();
- tracker.finished(lts);
+ tracker.endModification(lts);
}
@Override
@@ -157,7 +186,7 @@ public class MutatingVisitor extends GeneratingVisitor
if (retries > this.maxRetries)
throw new IllegalStateException(String.format("Can not execute statement %s after %d retries", statement, retries));
- sut.executeAsync(statement.cql(), SystemUnderTest.ConsistencyLevel.QUORUM, statement.bindings())
+ sut.executeAsync(statement.cql(), consistencyLevel, statement.bindings())
.whenComplete((res, t) -> {
if (t != null)
{
@@ -165,7 +194,8 @@ public class MutatingVisitor extends GeneratingVisitor
int delaySecs = 1;
executor.schedule(() -> executeAsyncWithRetries(future, statement, retries + 1), delaySecs, TimeUnit.SECONDS);
logger.info("Scheduled retry to happen with delay {} seconds", delaySecs);
- }else
+ }
+ else
future.complete(res);
});
}
diff --git a/harry-core/src/harry/visitors/RandomPartitionValidator.java b/harry-core/src/harry/visitors/RandomPartitionValidator.java
new file mode 100644
index 0000000..16cef9b
--- /dev/null
+++ b/harry-core/src/harry/visitors/RandomPartitionValidator.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.visitors;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import harry.core.Configuration;
+import harry.core.Run;
+import harry.model.Model;
+import harry.model.OpSelectors;
+import harry.operations.QueryGenerator;
+
+public class RandomPartitionValidator implements Visitor
+{
+ protected final Model model;
+ protected final QueryGenerator.TypedQueryGenerator queryGenerator;
+ protected final Run run;
+ protected final AtomicLong modifier;
+
+ public RandomPartitionValidator(Run run)
+ {
+ this(run, new Configuration.QuiescentCheckerConfig());
+ }
+
+ public RandomPartitionValidator(Run run,
+ Model.ModelFactory modelFactory)
+ {
+ this.model = modelFactory.make(run);
+ this.queryGenerator = new QueryGenerator.TypedQueryGenerator(run);
+ this.run = run;
+ this.modifier = new AtomicLong();
+ }
+
+ public void visit()
+ {
+ if (run.tracker.maxStarted() == 0)
+ return;
+
+ long modifier = this.modifier.incrementAndGet();
+ long pd = ((OpSelectors.DefaultPdSelector)run.pdSelector).randomVisitedPd(run.tracker.maxStarted(),
+ modifier,
+ run.schemaSpec);
+ model.validate(queryGenerator.inflate(run.pdSelector.maxLtsFor(pd), modifier));
+ }
+}
diff --git a/harry-integration/src/harry/visitors/SkippingVisitor.java b/harry-core/src/harry/visitors/SkippingVisitor.java
similarity index 100%
rename from harry-integration/src/harry/visitors/SkippingVisitor.java
rename to harry-core/src/harry/visitors/SkippingVisitor.java
diff --git a/harry-core/test/harry/operations/RelationTest.java b/harry-core/test/harry/operations/RelationTest.java
index eaa7ddb..178bd96 100644
--- a/harry-core/test/harry/operations/RelationTest.java
+++ b/harry-core/test/harry/operations/RelationTest.java
@@ -130,6 +130,11 @@ public class RelationTest
{
throw new RuntimeException("not implemented");
}
+
+ public long maxPosition(long maxLts)
+ {
+ throw new RuntimeException("not implemented");
+ }
},
new OpSelectors.DescriptorSelector()
{
diff --git a/harry-core/test/harry/runner/LockingDataTrackerTest.java b/harry-core/test/harry/runner/LockingDataTrackerTest.java
new file mode 100644
index 0000000..9cc5ce4
--- /dev/null
+++ b/harry-core/test/harry/runner/LockingDataTrackerTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.runner;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.LockSupport;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import harry.concurrent.ExecutorFactory;
+import harry.concurrent.Interruptible;
+import harry.concurrent.WaitQueue;
+import harry.ddl.SchemaGenerators;
+import harry.ddl.SchemaSpec;
+import harry.model.OpSelectors;
+
+import static harry.concurrent.InfiniteLoopExecutor.Daemon.NON_DAEMON;
+import static harry.concurrent.InfiniteLoopExecutor.Interrupts.UNSYNCHRONIZED;
+import static harry.concurrent.InfiniteLoopExecutor.SimulatorSafe.SAFE;
+import static harry.runner.Runner.merge;
+
+public class LockingDataTrackerTest
+{
+ @Test
+ public void testDataTracker() throws Throwable
+ {
+ SchemaSpec schemaSpec = SchemaGenerators.defaultSchemaSpecGen("test").inflate(1L);
+ OpSelectors.Rng rng = new OpSelectors.PCGFast(1L);
+ OpSelectors.PdSelector pdSelector = new OpSelectors.DefaultPdSelector(rng, 5, 2);
+ LockingDataTracker tracker = new LockingDataTracker(pdSelector, schemaSpec);
+
+ AtomicReference<State> excluded = new AtomicReference<>(State.UNLOCKED);
+ AtomicInteger readers = new AtomicInteger(0);
+ AtomicInteger writers = new AtomicInteger(0);
+
+ WaitQueue queue = WaitQueue.newWaitQueue();
+ WaitQueue.Signal interrupt = queue.register();
+ List<Throwable> errors = new CopyOnWriteArrayList<>();
+
+ long lts = 1;
+ long pd = pdSelector.pd(lts, schemaSpec);
+ for (int i = 0; i < 2; i++)
+ {
+ ExecutorFactory.Global.executorFactory().infiniteLoop("read-" + i, Runner.wrapInterrupt(state -> {
+ if (state == Interruptible.State.NORMAL)
+ {
+ tracker.beginModification(lts);
+ writers.incrementAndGet();
+ Assert.assertEquals(0, readers.get());
+ excluded.updateAndGet((prev) -> {
+ assert (prev == State.UNLOCKED || prev == State.LOCKED_FOR_WRITE) : prev;
+ return State.LOCKED_FOR_WRITE;
+ });
+ Assert.assertEquals(0, readers.get());
+ excluded.updateAndGet((prev) -> {
+ assert (prev == State.UNLOCKED || prev == State.LOCKED_FOR_WRITE) : prev;
+ return State.UNLOCKED;
+ });
+ Assert.assertEquals(0, readers.get());
+ writers.decrementAndGet();
+ tracker.endModification(lts);
+ LockSupport.parkNanos(100);
+ }
+ }, interrupt::signal, errors::add), SAFE, NON_DAEMON, UNSYNCHRONIZED);
+ }
+
+ for (int i = 0; i < 2; i++)
+ {
+ ExecutorFactory.Global.executorFactory().infiniteLoop("write-" + i, Runner.wrapInterrupt(state -> {
+ if (state == Interruptible.State.NORMAL)
+ {
+ tracker.beginValidation(pd);
+ readers.incrementAndGet();
+ Assert.assertEquals(0, writers.get());
+ excluded.updateAndGet((prev) -> {
+ assert (prev == State.UNLOCKED || prev == State.LOCKED_FOR_READ) : prev;
+ return State.LOCKED_FOR_READ;
+ });
+ Assert.assertEquals(0, writers.get());
+ excluded.updateAndGet((prev) -> {
+ assert (prev == State.UNLOCKED || prev == State.LOCKED_FOR_READ) : prev;
+ return State.UNLOCKED;
+ });
+ Assert.assertEquals(0, writers.get());
+ readers.decrementAndGet();
+ tracker.endValidation(pd);
+ LockSupport.parkNanos(100);
+ }
+ }, interrupt::signal, errors::add), SAFE, NON_DAEMON, UNSYNCHRONIZED);
+ }
+
+ interrupt.await(1, TimeUnit.MINUTES);
+ if (!errors.isEmpty())
+ throw merge(errors);
+ }
+ enum State { UNLOCKED, LOCKED_FOR_READ, LOCKED_FOR_WRITE }
+}
diff --git a/harry-integration-external/dependency-reduced-pom.xml b/harry-integration-external/dependency-reduced-pom.xml
index e2e00a5..23431b0 100644
--- a/harry-integration-external/dependency-reduced-pom.xml
+++ b/harry-integration-external/dependency-reduced-pom.xml
@@ -3,7 +3,7 @@
<parent>
<artifactId>harry-parent</artifactId>
<groupId>org.apache.cassandra</groupId>
- <version>0.0.1-SNAPSHOT</version>
+ <version>0.0.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>harry-integration-external</artifactId>
diff --git a/harry-integration-external/pom.xml b/harry-integration-external/pom.xml
index 9592b43..e0d4528 100755
--- a/harry-integration-external/pom.xml
+++ b/harry-integration-external/pom.xml
@@ -40,7 +40,7 @@
<dependency>
<groupId>org.apache.cassandra</groupId>
<artifactId>harry-core</artifactId>
- <version>${project.parent.version}</version>
+ <version>${project.parent.version}</version>
</dependency>
<dependency>
diff --git a/harry-integration/test/harry/model/TestBaseImpl.java b/harry-integration-external/src/com/datastax/driver/core/UtilsAccessor.java
similarity index 72%
rename from harry-integration/test/harry/model/TestBaseImpl.java
rename to harry-integration-external/src/com/datastax/driver/core/UtilsAccessor.java
index 2a91086..e22d125 100644
--- a/harry-integration/test/harry/model/TestBaseImpl.java
+++ b/harry-integration-external/src/com/datastax/driver/core/UtilsAccessor.java
@@ -16,17 +16,14 @@
* limitations under the License.
*/
-package harry.model;
+package com.datastax.driver.core;
-import org.junit.BeforeClass;
+import java.nio.ByteBuffer;
-import org.apache.cassandra.distributed.api.ICluster;
-
-// TODO: probably redundant
-public class TestBaseImpl extends org.apache.cassandra.distributed.test.TestBaseImpl
+public class UtilsAccessor
{
- @BeforeClass
- public static void beforeClass() throws Throwable {
- ICluster.setup();
+ public static long token(ByteBuffer bb)
+ {
+ return (long) com.datastax.driver.core.Token.getFactory("Murmur3Partitioner").hash(bb).getValue();
}
}
diff --git a/harry-integration-external/src/harry/model/sut/external/ExternalClusterSut.java b/harry-integration-external/src/harry/model/sut/external/ExternalClusterSut.java
index 76a4e3c..ae2808c 100644
--- a/harry-integration-external/src/harry/model/sut/external/ExternalClusterSut.java
+++ b/harry-integration-external/src/harry/model/sut/external/ExternalClusterSut.java
@@ -25,21 +25,24 @@ import com.fasterxml.jackson.annotation.JsonTypeName;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import harry.core.Configuration;
+import harry.ddl.SchemaSpec;
import harry.model.sut.SystemUnderTest;
+import harry.model.sut.TokenPlacementModel;
+import harry.util.ByteUtils;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
public class ExternalClusterSut implements SystemUnderTest
{
- public static void registerSubtypes()
- {
- Configuration.registerSubtypes(ExternalSutConfiguration.class);
- }
-
private final Session session;
private final ExecutorService executor;
@@ -54,6 +57,11 @@ public class ExternalClusterSut implements SystemUnderTest
this.executor = Executors.newFixedThreadPool(threads);
}
+ public Session session()
+ {
+ return session;
+ }
+
public static ExternalClusterSut create(ExternalSutConfiguration config)
{
// TODO: close Cluster and Session!
@@ -74,10 +82,10 @@ public class ExternalClusterSut implements SystemUnderTest
public void shutdown()
{
session.close();
- executor.shutdown();
+ executor.shutdownNow();
try
{
- executor.awaitTermination(60, TimeUnit.SECONDS);
+ executor.awaitTermination(10, TimeUnit.SECONDS);
}
catch (InterruptedException e)
{
@@ -107,8 +115,40 @@ public class ExternalClusterSut implements SystemUnderTest
}
}
+ public Object[][] executeNodeLocal(String statement, Predicate<Host> selector, Object... bindings)
+ {
+ int repeat = 10;
+ while (true)
+ {
+ try
+ {
+ Statement st = new SimpleStatement(statement, bindings);
+ boolean selected = false;
+ for (Host host : session.getCluster().getMetadata().getAllHosts())
+ {
+ if (selector.test(host))
+ {
+ st.setHost(host);
+ selected = true;
+ break;
+ }
+ }
+ assert selected;
+ return resultSetToObjectArray(session.execute(st));
+ }
+ catch (Throwable t)
+ {
+ if (repeat < 0)
+ throw t;
+
+ t.printStackTrace();
+ repeat--;
+ // retry unconditionally
+ }
+ }
+ }
- private static Object[][] resultSetToObjectArray(ResultSet rs)
+ public static Object[][] resultSetToObjectArray(ResultSet rs)
{
List<Row> rows = rs.all();
if (rows.size() == 0)
diff --git a/harry-integration-external/src/harry/model/sut/external/QuiescentLocalStateChecker.java b/harry-integration-external/src/harry/model/sut/external/QuiescentLocalStateChecker.java
new file mode 100644
index 0000000..bf4f949
--- /dev/null
+++ b/harry-integration-external/src/harry/model/sut/external/QuiescentLocalStateChecker.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.model.sut.external;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NavigableMap;
+
+import com.datastax.driver.core.Host;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.UtilsAccessor;
+import harry.core.Run;
+import harry.model.QuiescentLocalStateCheckerBase;
+import harry.model.sut.TokenPlacementModel;
+import harry.util.ByteUtils;
+
+import static harry.model.sut.TokenPlacementModel.Node;
+
+public class QuiescentLocalStateChecker extends QuiescentLocalStateCheckerBase
+{
+ public QuiescentLocalStateChecker(Run run)
+ {
+ this(run, 3);
+ }
+
+ public QuiescentLocalStateChecker(Run run, int rf)
+ {
+ super(run, rf);
+ }
+
+ @Override
+ protected NavigableMap<TokenPlacementModel.Range, List<TokenPlacementModel.Node>> getRing()
+ {
+ Session session = ((ExternalClusterSut) sut).session();
+ Host control = session.getState().getConnectedHosts().stream().findFirst().get();
+ SimpleStatement peers = new SimpleStatement("select peer, tokens from system.peers");
+ peers.setHost(control);
+ List<TokenPlacementModel.Node> other = TokenPlacementModel.peerStateToNodes(ExternalClusterSut.resultSetToObjectArray(session.execute(peers)));
+ SimpleStatement local = new SimpleStatement("select broadcast_address, tokens from system.local");
+ local.setHost(control);
+
+ List<TokenPlacementModel.Node> self = TokenPlacementModel.peerStateToNodes(ExternalClusterSut.resultSetToObjectArray(session.execute(local)));
+ List<TokenPlacementModel.Node> all = new ArrayList<>();
+ all.addAll(self);
+ all.addAll(other);
+ return TokenPlacementModel.replicate(all, rf);
+ }
+
+ @Override
+ protected long token(long pd)
+ {
+ return UtilsAccessor.token(ByteUtils.compose(ByteUtils.objectsToBytes(schema.inflatePartitionKey(pd))));
+ }
+
+
+ @Override
+ protected Object[][] executeNodeLocal(String statement, Node node, Object... bindings)
+ {
+ return ((ExternalClusterSut) sut).executeNodeLocal(statement,
+ (host) -> host.getAddress().toString().contains(node.id),
+ bindings);
+ }
+}
diff --git a/harry-integration-external/src/harry/runner/external/HarryRunnerExternal.java b/harry-integration-external/src/harry/runner/external/HarryRunnerExternal.java
index 621d579..3252860 100644
--- a/harry-integration-external/src/harry/runner/external/HarryRunnerExternal.java
+++ b/harry-integration-external/src/harry/runner/external/HarryRunnerExternal.java
@@ -28,8 +28,6 @@ import java.io.File;
public class HarryRunnerExternal extends HarryRunner {
public static void main(String[] args) throws Throwable {
- ExternalClusterSut.registerSubtypes();
-
HarryRunner runner = new HarryRunnerExternal();
File configFile = runner.loadConfig(args);
@@ -37,8 +35,11 @@ public class HarryRunnerExternal extends HarryRunner {
runner.run(configuration);
}
- @Override
- public void beforeRun(Runner.TimedRunner runner) {
+ public void beforeRun(Runner runner)
+ {
+ }
+ public void afterRun(Runner runner, Object result)
+ {
}
}
diff --git a/harry-integration-external/src/harry/runner/external/MiniStress.java b/harry-integration-external/src/harry/runner/external/MiniStress.java
new file mode 100644
index 0000000..5ad0574
--- /dev/null
+++ b/harry-integration-external/src/harry/runner/external/MiniStress.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.runner.external;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import harry.core.Configuration;
+import harry.core.Run;
+import harry.model.Model;
+import harry.model.OpSelectors;
+import harry.model.QuiescentChecker;
+import harry.model.sut.external.QuiescentLocalStateChecker;
+import harry.operations.Query;
+import harry.runner.HarryRunner;
+import harry.runner.LockingDataTracker;
+import harry.runner.Runner;
+import harry.visitors.AllPartitionsValidator;
+import harry.visitors.MutatingVisitor;
+import harry.visitors.Visitor;
+
+public class MiniStress
+{
+ public static void main(String... args) throws Throwable
+ {
+ File configFile = HarryRunner.loadConfig(args);
+ Configuration config = Configuration.fromFile(configFile);
+
+ List<Configuration.VisitorPoolConfiguration> concurrentPools = new ArrayList<>();
+ List<Visitor.VisitorFactory> postRunValidation = new ArrayList<>();
+ int duration = 60_000;
+ for (String arg : args)
+ {
+ if (arg.startsWith("--write"))
+ {
+ int concurrency = Integer.parseInt(arg.split("=")[1]);
+ concurrentPools.add(new Configuration.VisitorPoolConfiguration("Writer", concurrency, MutatingVisitor::new));
+ }
+ else if (arg.startsWith("--read"))
+ {
+ int concurrency = Integer.parseInt(arg.split("=")[1]);
+ AtomicLong modifier = new AtomicLong(0);
+ concurrentPools.add(new Configuration.VisitorPoolConfiguration("Reader", concurrency, (r) -> {
+ Model model = new QuiescentChecker(r);
+ OpSelectors.DefaultPdSelector pdSelector = (OpSelectors.DefaultPdSelector) r.pdSelector;
+
+ return () -> {
+ // Wait until we have something to validate
+ if (r.tracker.maxStarted() == 0)
+ return;
+
+ long randomPd = pdSelector.randomVisitedPd(r.tracker.maxStarted(), modifier.incrementAndGet(), r.schemaSpec);
+ Query query = Query.selectPartition(r.schemaSpec, randomPd, true);
+ model.validate(query);
+ query = Query.selectPartition(r.schemaSpec, randomPd, false);
+ model.validate(query);
+ };
+ }));
+ }
+ else if (arg.startsWith("--duration"))
+ {
+ duration = Integer.parseInt(arg.split("=")[1]);
+ }
+ else if (arg.startsWith("--validate-all-local"))
+ {
+ postRunValidation.add((r) -> new AllPartitionsValidator(r, 10, (lts) -> true, QuiescentLocalStateChecker::new));
+ }
+ else if (arg.startsWith("--validate-all"))
+ {
+ postRunValidation.add((r) -> new AllPartitionsValidator(r, 10, (lts) -> true, QuiescentChecker::new));
+ }
+ }
+
+ Configuration.ConfigurationBuilder builder = config.unbuild()
+ .setRunner(null);
+ config = builder.build();
+ Run run = config.createRun();
+
+ if (!(run.tracker instanceof LockingDataTracker))
+ {
+ System.err.println("Concurrent read/write workloads only work with a locking data tracker.");
+ System.exit(1);
+ }
+
+ System.out.println(Configuration.toYamlString(config));
+
+ int finalDuration = duration;
+ new Runner.ChainRunner(run, config,
+ Arrays.asList((r, c) -> new Runner.ConcurrentRunner(r, c, concurrentPools, finalDuration, TimeUnit.MILLISECONDS),
+ (r, c) -> new Runner.SingleVisitRunner(r, c, postRunValidation)))
+ .run();
+
+ run.sut.shutdown();
+
+ Configuration.toFile(new File(String.format("resume-%d.yaml", config.seed)),
+ config.unbuild()
+ .setClock(run.clock.toConfig())
+ .setDataTracker(run.tracker.toConfig())
+ .setDropSchema(false)
+ .setTruncateTable(false)
+ .setCreateSchema(false)
+ .build());
+ System.exit(0);
+ }
+
+}
\ No newline at end of file
diff --git a/harry-integration/pom.xml b/harry-integration/pom.xml
index 0531e2d..03daedc 100755
--- a/harry-integration/pom.xml
+++ b/harry-integration/pom.xml
@@ -65,6 +65,10 @@
<version>0.25</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
</dependencies>
<build>
<testResources>
diff --git a/harry-integration/src/harry/model/sut/ByteUtils.java b/harry-integration/src/harry/model/sut/ByteUtils.java
deleted file mode 100644
index ee7721c..0000000
--- a/harry-integration/src/harry/model/sut/ByteUtils.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package harry.model.sut;
-
-import java.net.InetAddress;
-import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.db.marshal.AsciiType;
-import org.apache.cassandra.db.marshal.BooleanType;
-import org.apache.cassandra.db.marshal.ByteType;
-import org.apache.cassandra.db.marshal.DoubleType;
-import org.apache.cassandra.db.marshal.FloatType;
-import org.apache.cassandra.db.marshal.InetAddressType;
-import org.apache.cassandra.db.marshal.IntegerType;
-import org.apache.cassandra.db.marshal.ListType;
-import org.apache.cassandra.db.marshal.SetType;
-import org.apache.cassandra.db.marshal.ShortType;
-import org.apache.cassandra.db.marshal.UUIDType;
-
-public class ByteUtils extends harry.util.ByteUtils
-{
- public static ByteBuffer objectToBytes(Object obj)
- {
- if (obj instanceof Integer)
- return bytes((int) obj);
- else if (obj instanceof Byte)
- return bytes((byte) obj);
- else if (obj instanceof Boolean)
- return bytes( (byte) (((boolean) obj) ? 1 : 0));
- else if (obj instanceof Short)
- return bytes((short) obj);
- else if (obj instanceof Long)
- return bytes((long) obj);
- else if (obj instanceof Float)
- return bytes((float) obj);
- else if (obj instanceof Double)
- return bytes((double) obj);
- else if (obj instanceof UUID)
- return bytes((UUID) obj);
- else if (obj instanceof InetAddress)
- return bytes((InetAddress) obj);
- else if (obj instanceof String)
- return bytes((String) obj);
- else if (obj instanceof List)
- {
- List l = (List) obj;
- if (l.isEmpty())
- return pack(Collections.emptyList(), 0);
- return ListType.getInstance(getType(l.get(0)), true).decompose(l);
- }
- else if (obj instanceof Set)
- {
- Set l = (Set) obj;
- if (l.isEmpty())
- return pack(Collections.emptyList(), 0);
- return SetType.getInstance(getType(l.iterator().next()), true).decompose(l);
- }
- else if (obj instanceof ByteBuffer)
- return (ByteBuffer) obj;
-
- else
- throw new IllegalArgumentException(String.format("Cannot convert value %s of type %s",
- obj,
- obj.getClass()));
- }
-
- public static AbstractType<?> getType(Object obj)
- {
- if (obj instanceof Integer)
- return IntegerType.instance; //TODO
- else if (obj instanceof Byte)
- return ByteType.instance;
- else if (obj instanceof Boolean)
- return BooleanType.instance;
- else if (obj instanceof Short)
- return ShortType.instance;
- else if (obj instanceof Long)
- return IntegerType.instance;
- else if (obj instanceof Float)
- return FloatType.instance;
- else if (obj instanceof Double)
- return DoubleType.instance;
- else if (obj instanceof UUID)
- return UUIDType.instance;
- else if (obj instanceof InetAddress)
- return InetAddressType.instance;
- else if (obj instanceof String)
- return AsciiType.instance;
- else
- throw new IllegalArgumentException(String.format("Cannot convert value %s of type %s",
- obj,
- obj.getClass()));
- }
-}
diff --git a/harry-integration/src/harry/model/sut/InJvmSut.java b/harry-integration/src/harry/model/sut/injvm/InJvmSutConfiguration.java
similarity index 50%
rename from harry-integration/src/harry/model/sut/InJvmSut.java
rename to harry-integration/src/harry/model/sut/injvm/InJvmSutConfiguration.java
index 687a5b3..fdb33f9 100644
--- a/harry-integration/src/harry/model/sut/InJvmSut.java
+++ b/harry-integration/src/harry/model/sut/injvm/InJvmSutConfiguration.java
@@ -16,43 +16,22 @@
* limitations under the License.
*/
-package harry.model.sut;
+package harry.model.sut.injvm;
import java.io.File;
import java.io.IOException;
-import java.util.Arrays;
import java.util.function.Consumer;
-import java.util.stream.Collectors;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
-import harry.core.Configuration;
import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.ICluster;
import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.distributed.api.IInvokableInstance;
-import org.apache.cassandra.locator.EndpointsForToken;
-import org.apache.cassandra.service.StorageService;
-public class InJvmSut extends InJvmSutBase<IInvokableInstance, Cluster>
-{
- public static void init()
- {
- Configuration.registerSubtypes(InJvmSutConfiguration.class);
- }
-
- public InJvmSut(Cluster cluster)
- {
- super(cluster, 10);
- }
-
- public InJvmSut(Cluster cluster, int threads)
- {
- super(cluster, threads);
- }
-
- @JsonTypeName("in_jvm")
- public static class InJvmSutConfiguration extends InJvmSutBaseConfiguration<IInvokableInstance, Cluster>
+@JsonTypeName("in_jvm")
+public class InJvmSutConfiguration extends InJvmSutBase.InJvmSutBaseConfiguration<IInvokableInstance, ICluster<IInvokableInstance>>
{
@JsonCreator
public InJvmSutConfiguration(@JsonProperty(value = "nodes", defaultValue = "3") int nodes,
@@ -62,13 +41,13 @@ public class InJvmSut extends InJvmSutBase<IInvokableInstance, Cluster>
super(nodes, worker_threads, root);
}
- protected Cluster cluster(Consumer<IInstanceConfig> cfg, int nodes, File root)
+ protected ICluster<IInvokableInstance> cluster(Consumer<IInstanceConfig> cfg, int nodes, File root)
{
try
{
return Cluster.build().withConfig(cfg)
- .withNodes(nodes)
- .withRoot(root)
+ .withNodes(nodes)
+ .withRoot(root)
.createWithoutStarting();
}
catch (IOException e)
@@ -77,22 +56,8 @@ public class InJvmSut extends InJvmSutBase<IInvokableInstance, Cluster>
}
}
- protected InJvmSutBase<IInvokableInstance, Cluster> sut(Cluster cluster)
+ protected InJvmSutBase<IInvokableInstance, ICluster<IInvokableInstance>> sut(ICluster<IInvokableInstance> cluster)
{
return new InJvmSut(cluster);
}
- }
-
- public int[] getReplicasFor(Object[] partitionKey, String keyspace, String table)
- {
- return cluster.get(1).appliesOnInstance((Object[] pk, String ks) ->
- {
- String pkString = Arrays.stream(pk).map(Object::toString).collect(Collectors.joining(":"));
- EndpointsForToken endpoints = StorageService.instance.getNaturalReplicasForToken(ks, table, pkString);
- int[] nodes = new int[endpoints.size()];
- for (int i = 0; i < endpoints.size(); i++)
- nodes[i] = endpoints.get(i).endpoint().address.getAddress()[3];
- return nodes;
- }).apply(partitionKey, keyspace);
- }
-}
+ }
\ No newline at end of file
diff --git a/harry-integration/src/harry/model/sut/MixedVersionInJvmSut.java b/harry-integration/src/harry/model/sut/injvm/MixedInJvmSutConfiguration.java
similarity index 60%
rename from harry-integration/src/harry/model/sut/MixedVersionInJvmSut.java
rename to harry-integration/src/harry/model/sut/injvm/MixedInJvmSutConfiguration.java
index 5ac03a7..dbead71 100644
--- a/harry-integration/src/harry/model/sut/MixedVersionInJvmSut.java
+++ b/harry-integration/src/harry/model/sut/injvm/MixedInJvmSutConfiguration.java
@@ -16,7 +16,8 @@
* limitations under the License.
*/
-package harry.model.sut;
+package harry.model.sut.injvm;
+
import java.io.File;
import java.io.IOException;
@@ -24,41 +25,17 @@ import java.util.ArrayList;
import java.util.List;
import java.util.function.Consumer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
-import harry.core.Configuration;
import org.apache.cassandra.distributed.UpgradeableCluster;
+import org.apache.cassandra.distributed.api.ICluster;
import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.distributed.api.IUpgradeableInstance;
import org.apache.cassandra.distributed.shared.Versions;
-/**
- * Make sure to set -Dcassandra.test.dtest_jar_path when using this class
- */
-public class MixedVersionInJvmSut extends InJvmSutBase<IUpgradeableInstance, UpgradeableCluster>
-{
- public static void init()
- {
- Configuration.registerSubtypes(MixedInJvmSutConfiguration.class);
- }
-
- private static final Logger logger = LoggerFactory.getLogger(MixedVersionInJvmSut.class);
- private final Versions.Version initialVersion;
- private final List<Versions.Version> versions;
-
- public MixedVersionInJvmSut(UpgradeableCluster cluster, Versions.Version initialVersion, List<Versions.Version> versions)
- {
- super(cluster, 10);
- this.initialVersion = initialVersion;
- this.versions = versions;
- }
-
- @JsonTypeName("mixed_in_jvm")
- public static class MixedInJvmSutConfiguration extends InJvmSutBaseConfiguration<IUpgradeableInstance, UpgradeableCluster>
+@JsonTypeName("mixed_in_jvm")
+public class MixedInJvmSutConfiguration extends InJvmSutBase.InJvmSutBaseConfiguration<IUpgradeableInstance, ICluster<IUpgradeableInstance>>
{
public final String initial_version;
public final List<String> versions;
@@ -85,7 +62,7 @@ public class MixedVersionInJvmSut extends InJvmSutBase<IUpgradeableInstance, Upg
upgradeVersions.add(allVersions.get(version));
}
- protected UpgradeableCluster cluster(Consumer<IInstanceConfig> cfg, int nodes, File root)
+ protected ICluster<IUpgradeableInstance> cluster(Consumer<IInstanceConfig> cfg, int nodes, File root)
{
try
{
@@ -102,37 +79,8 @@ public class MixedVersionInJvmSut extends InJvmSutBase<IUpgradeableInstance, Upg
}
}
- protected InJvmSutBase<IUpgradeableInstance, UpgradeableCluster> sut(UpgradeableCluster cluster)
+ protected InJvmSutBase<IUpgradeableInstance, ICluster<IUpgradeableInstance>> sut(ICluster<IUpgradeableInstance> cluster)
{
return new MixedVersionInJvmSut(cluster, initialVersion, upgradeVersions);
}
}
-
- @Override
- public void afterSchemaInit()
- {
- for (int i = 1; i <= cluster.size(); i++)
- {
- Versions.Version v = versions.get(i - 1);
- if (!v.equals(initialVersion))
- {
- logger.info("Upgrading {} node from {} to {}", i, initialVersion, v);
- IUpgradeableInstance instance = cluster.get(i);
- try
- {
- instance.shutdown().get();
- }
- catch (Throwable e)
- {
- throw new RuntimeException(e);
- }
- instance.setVersion(v);
- instance.startup();
- }
- else
- {
- logger.info("Skipping {} node upgrade, since it is already at the required version ({})", i, initialVersion);
- }
- }
- }
-}
\ No newline at end of file
diff --git a/harry-integration/src/harry/runner/Reproduce.java b/harry-integration/src/harry/runner/Reproduce.java
deleted file mode 100644
index 5d34ecd..0000000
--- a/harry-integration/src/harry/runner/Reproduce.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package harry.runner;
-
-import java.io.File;
-
-import harry.core.Configuration;
-import harry.core.Run;
-import harry.model.sut.PrintlnSut;
-import harry.operations.Query;
-import harry.reconciler.Reconciler;
-
-public class Reproduce
-{
-
- public static void main(String[] args) throws Throwable
- {
- File configFile = HarryRunner.loadConfig(args);
- Configuration configuration = Configuration.fromFile(configFile);
- System.out.println(Configuration.toYamlString(configuration));
- configuration = configuration.unbuild().setSUT(PrintlnSut::new).build();
-
- Run run = configuration.createRun();
-
- Reconciler reconciler = new Reconciler(run);
- long pd = 8135884698435133227L;
- System.out.println(reconciler.inflatePartitionState(pd,
- 5908L,
- Query.selectPartition(run.schemaSpec,
- pd,
- false))
- .toString(run.schemaSpec));
-
- // Try out everything you might want to try with a given run
- }
-}
diff --git a/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java b/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
index 0da7743..6c4d1d0 100644
--- a/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
+++ b/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
@@ -30,6 +30,9 @@ public class DataGeneratorsIntegrationTest extends CQLTester
public void testTimestampTieResolution() throws Throwable
{
Random rng = new Random(1);
+ String ks = "test_timestamp_tie_resolution";
+ createKeyspace(String.format("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}", ks));
+ int counter = 0;
for (ColumnSpec.DataType<?> dataType : new ColumnSpec.DataType[]{ ColumnSpec.int8Type,
ColumnSpec.int16Type,
ColumnSpec.int32Type,
@@ -38,7 +41,11 @@ public class DataGeneratorsIntegrationTest extends CQLTester
ColumnSpec.floatType,
ColumnSpec.doubleType })
{
- createTable(String.format("CREATE TABLE %%s (pk int PRIMARY KEY, v %s)",
+
+
+ String tbl = "table_" + (counter++);
+ createTable(String.format("CREATE TABLE %s.%s (pk int PRIMARY KEY, v %s)",
+ ks, tbl,
dataType.toString()));
for (int i = 0; i < 10_000; i++)
{
@@ -46,15 +53,15 @@ public class DataGeneratorsIntegrationTest extends CQLTester
long d2 = dataType.generator().adjustEntropyDomain(rng.nextLong());
for (long d : new long[]{ d1, d2 })
{
- execute("INSERT INTO %s (pk, v) VALUES (?,?) USING TIMESTAMP 1",
+ execute(String.format("INSERT INTO %s.%s (pk, v) VALUES (?,?) USING TIMESTAMP 1", ks,tbl),
i, dataType.generator().inflate(d));
}
if (dataType.compareLexicographically(d1, d2) > 0)
- assertRows(execute("SELECT v FROM %s WHERE pk=?", i),
+ assertRows(execute(String.format("SELECT v FROM %s.%s WHERE pk=?", ks, tbl), i),
row(dataType.generator().inflate(d1)));
else
- assertRows(execute("SELECT v FROM %s WHERE pk=?", i),
+ assertRows(execute(String.format("SELECT v FROM %s.%s WHERE pk=?", ks, tbl), i),
row(dataType.generator().inflate(d2)));
}
}
diff --git a/harry-integration/test/harry/model/ConcurrentQuiescentCheckerIntegrationTest.java b/harry-integration/test/harry/model/ConcurrentQuiescentCheckerIntegrationTest.java
new file mode 100644
index 0000000..e0fc67d
--- /dev/null
+++ b/harry-integration/test/harry/model/ConcurrentQuiescentCheckerIntegrationTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.model;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+import org.junit.Test;
+
+import harry.core.Configuration;
+import harry.ddl.SchemaGenerators;
+import harry.ddl.SchemaSpec;
+import harry.runner.LockingDataTracker;
+import harry.runner.Runner;
+import harry.visitors.MutatingVisitor;
+import harry.visitors.RandomPartitionValidator;
+
+import static harry.core.Configuration.VisitorPoolConfiguration.pool;
+import static java.util.Arrays.asList;
+
+public class ConcurrentQuiescentCheckerIntegrationTest extends ModelTestBase
+{
+ @Test
+ public void testConcurrentReadWriteWorkload() throws Throwable
+ {
+ Supplier<SchemaSpec> supplier = SchemaGenerators.progression(1);
+ int writeThreads = 2;
+ int readThreads = 2;
+
+ for (int i = 0; i < SchemaGenerators.GENERATORS_COUNT; i++)
+ {
+ SchemaSpec schema = supplier.get();
+
+ Configuration config = configuration(i, schema)
+ .setKeyspaceDdl("CREATE KEYSPACE IF NOT EXISTS " + schema.keyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};")
+ .setCreateSchema(true)
+ .setDropSchema(true)
+ .setDataTracker(LockingDataTracker::new)
+ .build();
+
+ Runner.concurrent(config,
+ asList(pool("Writer", writeThreads, MutatingVisitor::new),
+ pool("Reader", readThreads, RandomPartitionValidator::new)),
+ 2, TimeUnit.MINUTES)
+ .run();
+ }
+ }
+
+ @Override
+ Configuration.ModelConfiguration modelConfiguration()
+ {
+ return new Configuration.QuiescentCheckerConfig();
+ }
+}
\ No newline at end of file
diff --git a/harry-integration/test/harry/model/InJVMTokenAwareExecutorTest.java b/harry-integration/test/harry/model/InJVMTokenAwareExecutorTest.java
index c84c789..0ed596b 100644
--- a/harry-integration/test/harry/model/InJVMTokenAwareExecutorTest.java
+++ b/harry-integration/test/harry/model/InJVMTokenAwareExecutorTest.java
@@ -18,6 +18,7 @@
package harry.model;
+import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import org.junit.Before;
@@ -28,12 +29,12 @@ import harry.core.Configuration;
import harry.core.Run;
import harry.ddl.SchemaGenerators;
import harry.ddl.SchemaSpec;
-import harry.model.sut.InJVMTokenAwareVisitExecutor;
-import harry.model.sut.InJvmSut;
import harry.model.sut.SystemUnderTest;
-import harry.runner.RepairingLocalStateValidator;
+import harry.model.sut.injvm.InJVMTokenAwareVisitExecutor;
+import harry.model.sut.injvm.InJvmSut;
+import harry.runner.Runner;
+import harry.runner.UpToLtsRunner;
import harry.visitors.MutatingVisitor;
-import harry.visitors.Visitor;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.Feature;
@@ -58,7 +59,7 @@ public class InJVMTokenAwareExecutorTest extends IntegrationTestBase
}
@Test
- public void testRepair()
+ public void testRepair() throws Throwable
{
Supplier<SchemaSpec> schemaGen = SchemaGenerators.progression(1);
for (int cnt = 0; cnt < SchemaGenerators.DEFAULT_RUNS; cnt++)
@@ -70,21 +71,14 @@ public class InJVMTokenAwareExecutorTest extends IntegrationTestBase
Run run = configuration.createRun();
run.sut.schemaChange(run.schemaSpec.compile().cql());
- Visitor visitor = new MutatingVisitor(run, new InJVMTokenAwareVisitExecutor(run,
- new Configuration.MutatingRowVisitorConfiguration(),
- SystemUnderTest.ConsistencyLevel.NODE_LOCAL));
- OpSelectors.MonotonicClock clock = run.clock;
- long maxPd = 0;
- for (int i = 0; i < 10000; i++)
- {
- visitor.visit();
- maxPd = Math.max(maxPd, run.pdSelector.positionFor(clock.peek()));
- }
-
- RepairingLocalStateValidator validator = new RepairingLocalStateValidator(5, 1, run, new Configuration.QuiescentCheckerConfig());
- validator.visit();
+ Runner.chain(configuration,
+ UpToLtsRunner.factory(MutatingVisitor.factory(InJVMTokenAwareVisitExecutor.factory(new Configuration.MutatingRowVisitorConfiguration(),
+ SystemUnderTest.ConsistencyLevel.NODE_LOCAL,
+ 3)),
+ 10_000, 2, TimeUnit.SECONDS),
+ Runner.single(RepairingLocalStateValidator.factory(5, (v) -> true, QuiescentChecker::new)))
+ .run();
}
-
}
}
diff --git a/harry-integration/test/harry/model/IntegrationTestBase.java b/harry-integration/test/harry/model/IntegrationTestBase.java
index 6d8b4ef..9eb8a7e 100644
--- a/harry-integration/test/harry/model/IntegrationTestBase.java
+++ b/harry-integration/test/harry/model/IntegrationTestBase.java
@@ -28,8 +28,10 @@ import harry.core.Configuration;
import harry.ddl.SchemaGenerators;
import harry.ddl.SchemaSpec;
import harry.model.clock.OffsetClock;
-import harry.model.sut.InJvmSut;
+import harry.model.sut.injvm.InJvmSut;
+import harry.model.sut.injvm.InJvmSutBase;
import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.test.TestBaseImpl;
public class IntegrationTestBase extends TestBaseImpl
{
@@ -41,8 +43,9 @@ public class IntegrationTestBase extends TestBaseImpl
{
cluster = init(Cluster.build()
.withNodes(1)
+ .withConfig(InJvmSutBase.defaultConfig())
.start());
- sut = new InJvmSut(cluster, 1);
+ sut = new InJvmSut(cluster, 20);
}
@AfterClass
@@ -97,7 +100,7 @@ public class IntegrationTestBase extends TestBaseImpl
.setDropSchema(true)
.setSchemaProvider((seed1, sut) -> schema)
.setClusteringDescriptorSelector(sharedCDSelectorConfiguration().build())
- .setPartitionDescriptorSelector(new Configuration.DefaultPDSelectorConfiguration(1, 200))
+ .setPartitionDescriptorSelector(new Configuration.DefaultPDSelectorConfiguration(2, 200))
.setSUT(() -> sut);
}
}
\ No newline at end of file
diff --git a/harry-integration/test/harry/model/ModelTestBase.java b/harry-integration/test/harry/model/ModelTestBase.java
index 3381137..0baee11 100644
--- a/harry-integration/test/harry/model/ModelTestBase.java
+++ b/harry-integration/test/harry/model/ModelTestBase.java
@@ -18,6 +18,8 @@
package harry.model;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
import java.util.function.Function;
@@ -27,15 +29,17 @@ import harry.core.Configuration;
import harry.core.Run;
import harry.ddl.SchemaGenerators;
import harry.ddl.SchemaSpec;
-import harry.visitors.LoggingVisitor;
-import harry.visitors.MutatingRowVisitor;
import harry.runner.Runner;
+import harry.runner.UpToLtsRunner;
+import harry.visitors.MutatingRowVisitor;
+import harry.visitors.MutatingVisitor;
import harry.visitors.SingleValidator;
-import harry.visitors.Visitor;
public abstract class ModelTestBase extends IntegrationTestBase
{
- void negativeTest(Function<Run, Boolean> corrupt, BiConsumer<Throwable, Run> validate)
+ private final int ITERATIONS = 20_000;
+
+ void negativeTest(Function<Run, Boolean> corrupt, BiConsumer<Throwable, Run> validate) throws Throwable
{
Supplier<SchemaSpec> supplier = SchemaGenerators.progression(SchemaGenerators.DEFAULT_SWITCH_AFTER);
for (int i = 0; i < SchemaGenerators.DEFAULT_RUNS; i++)
@@ -62,18 +66,10 @@ public abstract class ModelTestBase extends IntegrationTestBase
Configuration config = builder.build();
Runner runner = config.createRunner();
- try
- {
- Run run = runner.getRun();
- beforeEach();
- run.sut.schemaChange(run.schemaSpec.compile().cql());
-
- runner.initAndStartAll().get(4, TimeUnit.MINUTES);
- }
- finally
- {
- runner.shutdown();
- }
+ Run run = runner.getRun();
+ beforeEach();
+ run.sut.schemaChange(run.schemaSpec.compile().cql());
+ runner.run();
}
}
@@ -89,37 +85,54 @@ public abstract class ModelTestBase extends IntegrationTestBase
return sharedConfiguration(seed, schema);
}
- void negativeTest(Function<Run, Boolean> corrupt, BiConsumer<Throwable, Run> validate, int counter, SchemaSpec schemaSpec)
+ void negativeTest(Function<Run, Boolean> corrupt, BiConsumer<Throwable, Run> validate, int counter, SchemaSpec schemaSpec) throws Throwable
{
- Configuration config = configuration(counter, schemaSpec).build();
+ Configuration config = configuration(counter, schemaSpec)
+ .setCreateSchema(true)
+ .setKeyspaceDdl(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': %d};",
+ schemaSpec.keyspace, cluster.size()))
+ .setDropSchema(true)
+ .build();
Run run = config.createRun();
- beforeEach();
- run.sut.schemaChange(run.schemaSpec.compile().cql());
- System.out.println(run.schemaSpec.compile().cql());
-
- Visitor visitor = new LoggingVisitor(run, MutatingRowVisitor::new);
- for (int i = 0; i < 20000; i++)
- visitor.visit();
+ new Runner.ChainRunner(run, config,
+ Arrays.asList(writer(ITERATIONS, 2, TimeUnit.MINUTES),
+ (r, c) -> new Runner.SingleVisitRunner(r, c, Collections.singletonList(this::validator)) {
+ @Override
+ public void runInternal()
+ {
+ if (!corrupt.apply(run))
+ {
+ System.out.println("Could not corrupt");
+ return;
+ }
+ try
+ {
+ super.runInternal();
+ throw new ShouldHaveThrownException();
+ }
+ catch (Throwable t)
+ {
+ validate.accept(t, run);
+ }
+ }
+ })).run();
+ }
- SingleValidator validator = validator(run);
- validator.visit(0);
+ public static Configuration.RunnerConfiguration writer(long iterations, int runtime, TimeUnit timeUnit)
+ {
+ return (run, config) -> {
+ return new UpToLtsRunner(run, config,
+ Collections.singletonList((r_) -> new MutatingVisitor(r_, MutatingRowVisitor::new)),
+ iterations,
+ runtime, timeUnit);
+ };
+ }
- if (!corrupt.apply(run))
- {
- System.out.println("Could not corrupt");
- return;
- }
+ public static class ShouldHaveThrownException extends AssertionError
+ {
- try
- {
- validator.visit(0);
- }
- catch (Throwable t)
- {
- validate.accept(t, run);
- }
}
}
diff --git a/harry-integration/test/harry/model/QuerySelectorNegativeTest.java b/harry-integration/test/harry/model/QuerySelectorNegativeTest.java
index 13ae548..95bcbc8 100644
--- a/harry-integration/test/harry/model/QuerySelectorNegativeTest.java
+++ b/harry-integration/test/harry/model/QuerySelectorNegativeTest.java
@@ -78,6 +78,7 @@ public class QuerySelectorNegativeTest extends IntegrationTestBase
HideRowCorruptor::new),
(run) -> new AddExtraRowCorruptor(run.schemaSpec,
run.clock,
+ run.tracker,
run.descriptorSelector)
);
}
diff --git a/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java b/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
index 97a554a..9ce991b 100644
--- a/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
+++ b/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
@@ -45,18 +45,25 @@ import harry.visitors.SingleValidator;
public class QuiescentCheckerIntegrationTest extends ModelTestBase
{
+ private final long CORRUPT_LTS = 0L;
+
@Override
protected SingleValidator validator(Run run)
{
- return new SingleValidator(100, run, modelConfiguration());
+ return new SingleValidator(100, run, modelConfiguration()) {
+ public void visit()
+ {
+ visit(CORRUPT_LTS);
+ }
+ };
}
@Test
- public void testNormalCondition()
+ public void testNormalCondition() throws Throwable
{
negativeTest((run) -> true,
(t, run) -> {
- if (t != null)
+ if (!(t instanceof ShouldHaveThrownException))
throw new AssertionError(String.format("Throwable was supposed to be null. Schema: %s",
run.schemaSpec.compile().cql()),
t);
@@ -82,7 +89,7 @@ public class QuiescentCheckerIntegrationTest extends ModelTestBase
Model.ModelFactory factory = modelConfiguration();
ConcurrentRunnerConfig concurrent =
- new ConcurrentRunnerConfig(4, Collections.singletonList(new LoggingVisitorConfiguration(new Configuration.MutatingRowVisitorConfiguration())),
+ new ConcurrentRunnerConfig(Arrays.asList(new Configuration.VisitorPoolConfiguration("Writer", 4, new Configuration.MutatingVisitorConfiguation(new Configuration.MutatingRowVisitorConfiguration()))),
30, TimeUnit.SECONDS);
SingleVisitRunnerConfig sequential =
new SingleVisitRunnerConfig(Collections.singletonList(new RecentPartitionsValidatorConfiguration(1024, 0, 1, factory::make)));
@@ -93,7 +100,7 @@ public class QuiescentCheckerIntegrationTest extends ModelTestBase
}
@Test
- public void testDetectsMissingRow()
+ public void testDetectsMissingRow() throws Throwable
{
negativeTest((run) -> {
SimpleQueryResponseCorruptor corruptor = new SimpleQueryResponseCorruptor(run.schemaSpec,
@@ -101,7 +108,7 @@ public class QuiescentCheckerIntegrationTest extends ModelTestBase
HideRowCorruptor::new);
Query query = Query.selectPartition(run.schemaSpec,
- run.pdSelector.pd(0, run.schemaSpec),
+ run.pdSelector.pd(CORRUPT_LTS, run.schemaSpec),
false);
return corruptor.maybeCorrupt(query, run.sut);
@@ -120,23 +127,24 @@ public class QuiescentCheckerIntegrationTest extends ModelTestBase
}
@Test
- public void testDetectsExtraRow()
+ public void testDetectsExtraRow() throws Throwable
{
negativeTest((run) -> {
QueryResponseCorruptor corruptor = new AddExtraRowCorruptor(run.schemaSpec,
run.clock,
+ run.tracker,
run.descriptorSelector);
return corruptor.maybeCorrupt(Query.selectPartition(run.schemaSpec,
- run.pdSelector.pd(0, run.schemaSpec),
+ run.pdSelector.pd(CORRUPT_LTS, run.schemaSpec),
false),
run.sut);
},
(t, run) -> {
String expected = "Found a row in the model that is not present in the resultset";
String expected2 = "Expected results to have the same number of results, but actual result iterator has more results";
-
- if (t.getMessage().contains(expected) || t.getMessage().contains(expected2))
+ String expected3 = "Found a row while model predicts statics only";
+ if (t.getMessage().contains(expected) || t.getMessage().contains(expected2) || t.getMessage().contains(expected3))
return;
throw new AssertionError(String.format("Exception string mismatch.\nExpected error: %s.\nActual error: %s", expected, t.getMessage()),
@@ -146,7 +154,7 @@ public class QuiescentCheckerIntegrationTest extends ModelTestBase
@Test
- public void testDetectsRemovedColumn()
+ public void testDetectsRemovedColumn() throws Throwable
{
negativeTest((run) -> {
SimpleQueryResponseCorruptor corruptor = new SimpleQueryResponseCorruptor(run.schemaSpec,
@@ -154,7 +162,7 @@ public class QuiescentCheckerIntegrationTest extends ModelTestBase
HideValueCorruptor::new);
return corruptor.maybeCorrupt(Query.selectPartition(run.schemaSpec,
- run.pdSelector.pd(0, run.schemaSpec),
+ run.pdSelector.pd(CORRUPT_LTS, run.schemaSpec),
false),
run.sut);
},
@@ -172,7 +180,7 @@ public class QuiescentCheckerIntegrationTest extends ModelTestBase
@Test
- public void testDetectsOverwrittenRow()
+ public void testDetectsOverwrittenRow() throws Throwable
{
negativeTest((run) -> {
SimpleQueryResponseCorruptor corruptor = new SimpleQueryResponseCorruptor(run.schemaSpec,
@@ -180,7 +188,7 @@ public class QuiescentCheckerIntegrationTest extends ModelTestBase
ChangeValueCorruptor::new);
return corruptor.maybeCorrupt(Query.selectPartition(run.schemaSpec,
- run.pdSelector.pd(0, run.schemaSpec),
+ run.pdSelector.pd(CORRUPT_LTS, run.schemaSpec),
false),
run.sut);
},
diff --git a/harry-integration/test/harry/model/QuiescentLocalStateCheckerIntegrationTest.java b/harry-integration/test/harry/model/QuiescentLocalStateCheckerIntegrationTest.java
new file mode 100644
index 0000000..ae537db
--- /dev/null
+++ b/harry-integration/test/harry/model/QuiescentLocalStateCheckerIntegrationTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.model;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import harry.core.Configuration;
+import harry.ddl.SchemaGenerators;
+import harry.ddl.SchemaSpec;
+import harry.model.sut.injvm.InJvmSut;
+import harry.model.sut.injvm.InJvmSutBase;
+import harry.model.sut.injvm.QuiescentLocalStateChecker;
+import harry.runner.Runner;
+import harry.visitors.AllPartitionsValidator;
+import harry.visitors.MutatingVisitor;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.Feature;
+
+public class QuiescentLocalStateCheckerIntegrationTest extends ModelTestBase
+{
+ @BeforeClass
+ public static void before() throws Throwable
+ {
+ cluster = init(Cluster.build()
+ .withNodes(2)
+ .withConfig(InJvmSutBase.defaultConfig().andThen((cfg) -> cfg.with(Feature.GOSSIP)))
+ .start());
+ sut = new InJvmSut(cluster, 1);
+ }
+
+
+ @Test
+ public void testQuiescentLocalStateChecker() throws Throwable
+ {
+ Supplier<SchemaSpec> supplier = SchemaGenerators.progression(1);
+ for (int i = 0; i < SchemaGenerators.GENERATORS_COUNT; i++)
+ {
+ SchemaSpec schema = supplier.get();
+
+ Configuration config = configuration(i, schema)
+ .setKeyspaceDdl("CREATE KEYSPACE IF NOT EXISTS " + schema.keyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};")
+ .setCreateSchema(true)
+ .setDropSchema(true)
+ .build();
+
+ Runner.chain(config,
+ Runner.sequential(MutatingVisitor::new, 2, TimeUnit.SECONDS),
+ Runner.single(AllPartitionsValidator.factory(1, (p) -> true, QuiescentLocalStateChecker.factory(1))))
+ .run();
+ break;
+ }
+ }
+
+ @Override
+ Configuration.ModelConfiguration modelConfiguration()
+ {
+ return new Configuration.QuiescentCheckerConfig();
+ }
+}
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index c044783..c749d67 100755
--- a/pom.xml
+++ b/pom.xml
@@ -156,7 +156,7 @@
<dependency>
<groupId>org.reflections</groupId>
<artifactId>reflections</artifactId>
- <version>0.9.12</version>
+ <version>0.10.2</version>
</dependency>
<dependency>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org