You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by if...@apache.org on 2021/07/15 07:27:44 UTC
[cassandra-harry] branch CASSANDRA-16262-2 created (now af4bb14)
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a change to branch CASSANDRA-16262-2
in repository https://gitbox.apache.org/repos/asf/cassandra-harry.git.
at af4bb14 Move classes to appropriate packages
This branch includes the following new commits:
new f22d52e Core improvements
new 57053de Integration improvements
new 32a9200 Adjust config files
new af4bb14 Move classes to appropriate packages
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra-harry] 03/04: Adjust config files
Posted by if...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch CASSANDRA-16262-2
in repository https://gitbox.apache.org/repos/asf/cassandra-harry.git
commit 32a9200e706883e8e248c81f95174fa34226e90c
Author: Alex Petrov <ol...@gmail.com>
AuthorDate: Tue Jul 13 10:41:18 2021 +0200
Adjust config files
---
README.md | 3 --
conf/{example.yaml => default.yaml} | 63 ++++++++++++++++++++++++-------------
conf/external.yaml | 51 +++++++++++++++++++++---------
docker/Dockerfile.local | 2 +-
docker/run.sh | 2 +-
pom.xml | 24 ++++++--------
run-jvm.sh | 2 +-
7 files changed, 90 insertions(+), 57 deletions(-)
diff --git a/README.md b/README.md
index ec93b22..841e4ea 100644
--- a/README.md
+++ b/README.md
@@ -491,10 +491,8 @@ Harry is by no means feature-complete. Main things that are missing are:
* Some types (such as collections) are not deflatable
* Some types are implemented, but are not hooked up (`blob` and `text`) to DSL/generator
- * Partition deletions are not implemented
* 2i queries are not implemented
* Compact storage is not implemented
- * Static columns are not implemented
* Fault injection is not implemented
* Runner and scheduler are rather rudimentary and require significant rework and proper scheduling
* TTL is not supported
@@ -508,7 +506,6 @@ Some things, even though are implemented, can be improved or optimized:
* Inflated partition state and per-row operation log should be done in a compact
off-heap data structure
* Exhaustive checker can be significantly optimized
- * Harry shouldn't rely on java-driver for query generation
* Exhaustive checker should use more precise information from data tracker, not
just watermarks
* Decision-making about _when_ we visit partitions and/or rows should be improved
diff --git a/conf/example.yaml b/conf/default.yaml
similarity index 76%
rename from conf/example.yaml
rename to conf/default.yaml
index 3a3c5ab..cdbc34c 100644
--- a/conf/example.yaml
+++ b/conf/default.yaml
@@ -35,14 +35,6 @@ clock:
epoch_length: 1
epoch_time_unit: "SECONDS"
-# Runner is a is a component that schedules operations that change the cluster (system under test)
-# and model state.
-runner:
- concurrent:
- writer_threads: 2
- round_robin_validator_threads: 1
- recent_partition_validator_threads: 1
-
run_time: 2
run_time_unit: "HOURS"
@@ -54,12 +46,6 @@ system_under_test:
worker_threads: 10
root: "/tmp/harry/"
-# Model is responsible for tracking logical timestamps that
-model:
- exhaustive_checker:
- max_seen_lts: 19
- max_complete_lts: 16
-
# Partition descriptor selector controls how partitions is selected based on the current logical
# timestamp. Default implementation is a sliding window of partition descriptors that will visit
# one partition after the other in the window `slide_after_repeats` times. After that will
@@ -77,18 +63,53 @@ clustering_descriptor_selector:
default:
modifications_per_lts:
type: "constant"
- constant: 10
+ constant: 4
rows_per_modification:
type: "constant"
- constant: 10
+ constant: 2
operation_kind_weights:
- WRITE: 97
DELETE_RANGE: 1
+ DELETE_SLICE: 1
DELETE_ROW: 1
DELETE_COLUMN: 1
+ DELETE_PARTITION: 1
+ DELETE_COLUMN_WITH_STATICS: 1
+ INSERT_WITH_STATICS: 50
+ INSERT: 50
+ UPDATE_WITH_STATICS: 50
+ UPDATE: 50
column_mask_bitsets: null
- max_partition_size: 100
+ max_partition_size: 1000
+
+# Runner is a is a component that schedules operations that change the cluster (system under test)
+# and model state.
+runner:
+ sequential:
+ partition_visitors:
+ - logging:
+ row_visitor:
+ mutating: {}
+ - sampler:
+ trigger_after: 1000
+ sample_partitions: 10
+ - parallel_validate_recent_partitions:
+ partition_count: 100
+ queries_per_partition: 2
+ concurrency: 20
+ trigger_after: 10000
+ model:
+ quiescent_checker: {}
+ - validate_all_partitions:
+ concurrency: 20
+ trigger_after: 100000
+ model:
+ quiescent_checker: {}
+
+# Model is responsible for tracking logical timestamps that
+data_tracker:
+ default:
+ max_seen_lts: -1
+ max_complete_lts: -1
-# Default Row Visitor
-row_visitor:
- default: {}
\ No newline at end of file
+metric_reporter:
+ no_op: {}
\ No newline at end of file
diff --git a/conf/external.yaml b/conf/external.yaml
index 770dda9..52943b8 100644
--- a/conf/external.yaml
+++ b/conf/external.yaml
@@ -35,14 +35,6 @@ clock:
epoch_length: 1
epoch_time_unit: "SECONDS"
-# Runner is a is a component that schedules operations that change the cluster (system under test)
-# and model state.
-runner:
- concurrent:
- writer_threads: 2
- round_robin_validator_threads: 1
- recent_partition_validator_threads: 1
-
run_time: 2
run_time_unit: "HOURS"
@@ -78,18 +70,47 @@ clustering_descriptor_selector:
default:
modifications_per_lts:
type: "constant"
- constant: 10
+ constant: 4
rows_per_modification:
type: "constant"
- constant: 10
+ constant: 2
operation_kind_weights:
- WRITE: 97
DELETE_RANGE: 1
+ DELETE_SLICE: 1
DELETE_ROW: 1
DELETE_COLUMN: 1
+ DELETE_PARTITION: 1
+ DELETE_COLUMN_WITH_STATICS: 1
+ INSERT_WITH_STATICS: 50
+ INSERT: 50
+ UPDATE_WITH_STATICS: 50
+ UPDATE: 50
column_mask_bitsets: null
- max_partition_size: 100
+ max_partition_size: 1000
+
+# Runner is a is a component that schedules operations that change the cluster (system under test)
+# and model state.
+runner:
+ sequential:
+ partition_visitors:
+ - logging:
+ row_visitor:
+ mutating: {}
+ - sampler:
+ trigger_after: 1000
+ sample_partitions: 10
+ - parallel_validate_recent_partitions:
+ partition_count: 100
+ queries_per_partition: 2
+ concurrency: 20
+ trigger_after: 10000
+ model:
+ quiescent_checker: {}
+ - validate_all_partitions:
+ concurrency: 20
+ trigger_after: 100000
+ model:
+ quiescent_checker: {}
-# Default Row Visitor
-row_visitor:
- default: {}
\ No newline at end of file
+metric_reporter:
+ no_op: {}
\ No newline at end of file
diff --git a/docker/Dockerfile.local b/docker/Dockerfile.local
index 21ce8fc..4b0032f 100644
--- a/docker/Dockerfile.local
+++ b/docker/Dockerfile.local
@@ -28,7 +28,7 @@ COPY ./harry-core/target/*.jar /opt/harry/lib/
COPY ./harry-integration/target/lib/ /opt/harry/lib/
COPY ./harry-integration/target/*.jar /opt/harry/
COPY ./test/conf/logback-dtest.xml /opt/harry/test/conf/logback-dtest.xml
-COPY ./conf/example.yaml /opt/harry/example.yaml
+COPY ./conf/default.yaml /opt/harry/default.yaml
COPY ./docker/run.sh /opt/harry/
WORKDIR /opt/harry
diff --git a/docker/run.sh b/docker/run.sh
index 490b21f..48afa0b 100755
--- a/docker/run.sh
+++ b/docker/run.sh
@@ -75,7 +75,7 @@ while true; do
-cp /opt/harry/lib/*:/opt/harry/harry-integration-0.0.1-SNAPSHOT.jar \
-Dharry.root=${HARRY_DIR} \
harry.runner.HarryRunnerJvm \
- /opt/harry/example.yaml
+ /opt/harry/default.yaml
if [ $? -ne 0 ]; then
if [ -e "failure.dump" ]; then
diff --git a/pom.xml b/pom.xml
index 7e23efd..276a670 100755
--- a/pom.xml
+++ b/pom.xml
@@ -52,7 +52,7 @@
<properties>
<javac.target>1.8</javac.target>
<harry.version>0.0.1-SNAPSHOT</harry.version>
- <cassandra.version>4.0.0-SNAPSHOT</cassandra.version>
+ <cassandra.version>4.1-58515c2de6</cassandra.version>
<jackson.version>2.11.3</jackson.version>
<dtest.version>0.0.7</dtest.version>
<jmh.version>1.11.3</jmh.version>
@@ -135,28 +135,22 @@
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>27.0-jre</version>
- </dependency>
-
- <dependency>
<groupId>org.apache.cassandra</groupId>
<artifactId>cassandra-dtest-shaded</artifactId>
<version>${cassandra.version}</version>
</dependency>
<dependency>
- <groupId>com.datastax.cassandra</groupId>
- <artifactId>cassandra-driver-core</artifactId>
- <version>3.6.0</version>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ <version>0.9.12</version>
</dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- <version>3.1</version>
- </dependency>
+ <dependency>
+ <groupId>com.datastax.cassandra</groupId>
+ <artifactId>cassandra-driver-core</artifactId>
+ <version>3.6.0</version>
+ </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
diff --git a/run-jvm.sh b/run-jvm.sh
index 6c330f8..e0999b5 100755
--- a/run-jvm.sh
+++ b/run-jvm.sh
@@ -46,4 +46,4 @@ java -ea \
-Dorg.apache.cassandra.test.logback.configurationFile=file://test/conf/logback-dtest.xml \
-cp harry-integration/target/harry-integration-0.0.1-SNAPSHOT.jar:$(find harry-integration/target/dependency/*.jar | tr -s '\n' ':'). \
harry.runner.HarryRunnerJvm \
- conf/example.yaml
\ No newline at end of file
+ conf/default.yaml
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra-harry] 04/04: Move classes to appropriate packages
Posted by if...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch CASSANDRA-16262-2
in repository https://gitbox.apache.org/repos/asf/cassandra-harry.git
commit af4bb1488f4c748a4e430ec3f105feb26820fd26
Author: Alex Petrov <ol...@gmail.com>
AuthorDate: Thu Jul 15 08:29:57 2021 +0200
Move classes to appropriate packages
---
harry-core/src/harry/core/Configuration.java | 20 ++++----
harry-core/src/harry/core/Run.java | 2 +-
.../src/harry/corruptor/AddExtraRowCorruptor.java | 2 +-
.../harry/corruptor/QueryResponseCorruptor.java | 3 +-
harry-core/src/harry/model/Model.java | 3 +-
harry-core/src/harry/model/NoOpChecker.java | 2 +-
harry-core/src/harry/model/QuiescentChecker.java | 4 +-
harry-core/src/harry/model/SelectHelper.java | 2 +-
.../src/harry/{runner => operations}/Query.java | 5 +-
.../{runner => operations}/QueryGenerator.java | 4 +-
harry-core/src/harry/reconciler/Reconciler.java | 8 ++--
harry-core/src/harry/runner/Runner.java | 2 +-
.../AbstractPartitionVisitor.java | 7 ++-
.../AllPartitionsValidator.java | 3 +-
.../CorruptingPartitionVisitor.java | 4 +-
.../LoggingPartitionVisitor.java | 2 +-
.../MutatingPartitionVisitor.java | 4 +-
.../{runner => visitors}/MutatingRowVisitor.java | 4 +-
.../src/harry/{runner => visitors}/Operation.java | 2 +-
.../ParallelRecentPartitionValidator.java | 4 +-
.../{runner => visitors}/ParallelValidator.java | 2 +-
.../{runner => visitors}/PartitionVisitor.java | 2 +-
.../RecentPartitionValidator.java | 6 +--
.../src/harry/{runner => visitors}/Sampler.java | 2 +-
.../SinglePartitionValidator.java | 5 +-
harry-core/test/harry/model/OpSelectorsTest.java | 6 +--
harry-core/test/harry/operations/RelationTest.java | 2 -
.../dependency-reduced-pom.xml | 54 ++++++++++++++++++++++
.../runner/FaultInjectingPartitionVisitor.java | 3 ++
.../src/harry/runner/QueryingNoOpChecker.java | 1 +
.../harry/runner/RepairingLocalStateValidator.java | 3 ++
harry-integration/src/harry/runner/Reproduce.java | 1 +
.../src/harry/runner/TrivialShrinker.java | 53 ++-------------------
.../harry/visitors/SkippingPartitionVisitor.java | 53 +++++++++++++++++++++
.../test/harry/ddl/SchemaGenTest.java | 2 +-
.../generators/DataGeneratorsIntegrationTest.java | 8 ++--
.../test/harry/model/ModelTestBase.java | 8 ++--
.../harry/model/QuerySelectorNegativeTest.java | 10 ++--
.../test/harry/model/QuerySelectorTest.java | 10 ++--
.../model/QuiescentCheckerIntegrationTest.java | 6 +--
.../test/harry/model/TestEveryClustering.java | 24 ++++------
.../test/harry/op/RowVisitorTest.java | 2 +-
42 files changed, 208 insertions(+), 142 deletions(-)
diff --git a/harry-core/src/harry/core/Configuration.java b/harry-core/src/harry/core/Configuration.java
index 4a2a1e6..0e065d5 100644
--- a/harry-core/src/harry/core/Configuration.java
+++ b/harry-core/src/harry/core/Configuration.java
@@ -46,19 +46,19 @@ import harry.model.clock.ApproximateMonotonicClock;
import harry.model.clock.OffsetClock;
import harry.model.sut.PrintlnSut;
import harry.model.sut.SystemUnderTest;
-import harry.runner.AllPartitionsValidator;
-import harry.runner.CorruptingPartitionVisitor;
+import harry.visitors.AllPartitionsValidator;
+import harry.visitors.CorruptingPartitionVisitor;
import harry.runner.DataTracker;
import harry.runner.DefaultDataTracker;
-import harry.runner.LoggingPartitionVisitor;
-import harry.runner.MutatingPartitionVisitor;
-import harry.runner.MutatingRowVisitor;
-import harry.runner.Operation;
-import harry.runner.ParallelRecentPartitionValidator;
-import harry.runner.PartitionVisitor;
-import harry.runner.RecentPartitionValidator;
+import harry.visitors.LoggingPartitionVisitor;
+import harry.visitors.MutatingPartitionVisitor;
+import harry.visitors.MutatingRowVisitor;
+import harry.visitors.Operation;
+import harry.visitors.ParallelRecentPartitionValidator;
+import harry.visitors.PartitionVisitor;
+import harry.visitors.RecentPartitionValidator;
import harry.runner.Runner;
-import harry.runner.Sampler;
+import harry.visitors.Sampler;
import harry.util.BitSet;
public class Configuration
diff --git a/harry-core/src/harry/core/Run.java b/harry-core/src/harry/core/Run.java
index d31e7ad..b0c8afc 100644
--- a/harry-core/src/harry/core/Run.java
+++ b/harry-core/src/harry/core/Run.java
@@ -22,7 +22,7 @@ import harry.ddl.SchemaSpec;
import harry.model.OpSelectors;
import harry.model.sut.SystemUnderTest;
import harry.runner.DataTracker;
-import harry.runner.QueryGenerator;
+import harry.operations.QueryGenerator;
public class Run
{
diff --git a/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java b/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java
index 65f4a0d..e05dffd 100644
--- a/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java
+++ b/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java
@@ -30,7 +30,7 @@ import harry.model.OpSelectors;
import harry.model.SelectHelper;
import harry.model.sut.SystemUnderTest;
import harry.operations.WriteHelper;
-import harry.runner.Query;
+import harry.operations.Query;
public class AddExtraRowCorruptor implements QueryResponseCorruptor
{
diff --git a/harry-core/src/harry/corruptor/QueryResponseCorruptor.java b/harry-core/src/harry/corruptor/QueryResponseCorruptor.java
index f4a3778..62bf589 100644
--- a/harry-core/src/harry/corruptor/QueryResponseCorruptor.java
+++ b/harry-core/src/harry/corruptor/QueryResponseCorruptor.java
@@ -31,8 +31,7 @@ import harry.model.OpSelectors;
import harry.model.SelectHelper;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
-import harry.runner.HarryRunner;
-import harry.runner.Query;
+import harry.operations.Query;
public interface QueryResponseCorruptor
{
diff --git a/harry-core/src/harry/model/Model.java b/harry-core/src/harry/model/Model.java
index d6ab865..1521d0f 100644
--- a/harry-core/src/harry/model/Model.java
+++ b/harry-core/src/harry/model/Model.java
@@ -19,8 +19,7 @@
package harry.model;
import harry.core.Run;
-import harry.reconciler.Reconciler;
-import harry.runner.Query;
+import harry.operations.Query;
public interface Model
{
diff --git a/harry-core/src/harry/model/NoOpChecker.java b/harry-core/src/harry/model/NoOpChecker.java
index a13b6ec..10f0a4a 100644
--- a/harry-core/src/harry/model/NoOpChecker.java
+++ b/harry-core/src/harry/model/NoOpChecker.java
@@ -20,7 +20,7 @@ package harry.model;
import harry.core.Run;
import harry.model.sut.SystemUnderTest;
-import harry.runner.Query;
+import harry.operations.Query;
public class NoOpChecker implements Model
{
diff --git a/harry-core/src/harry/model/QuiescentChecker.java b/harry-core/src/harry/model/QuiescentChecker.java
index a48ccce..d17d6f4 100644
--- a/harry-core/src/harry/model/QuiescentChecker.java
+++ b/harry-core/src/harry/model/QuiescentChecker.java
@@ -24,15 +24,13 @@ import java.util.Iterator;
import java.util.List;
import java.util.function.Supplier;
-import harry.core.Configuration;
import harry.core.Run;
import harry.data.ResultSetRow;
import harry.ddl.SchemaSpec;
import harry.model.sut.SystemUnderTest;
import harry.reconciler.Reconciler;
import harry.runner.DataTracker;
-import harry.runner.Query;
-import harry.runner.QueryGenerator;
+import harry.operations.Query;
import static harry.generators.DataGenerators.NIL_DESCR;
diff --git a/harry-core/src/harry/model/SelectHelper.java b/harry-core/src/harry/model/SelectHelper.java
index fc8f6f7..c0813ba 100644
--- a/harry-core/src/harry/model/SelectHelper.java
+++ b/harry-core/src/harry/model/SelectHelper.java
@@ -28,7 +28,7 @@ import harry.ddl.SchemaSpec;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
import harry.operations.Relation;
-import harry.runner.Query;
+import harry.operations.Query;
public class SelectHelper
{
diff --git a/harry-core/src/harry/runner/Query.java b/harry-core/src/harry/operations/Query.java
similarity index 98%
rename from harry-core/src/harry/runner/Query.java
rename to harry-core/src/harry/operations/Query.java
index 1eef505..d7e75c3 100644
--- a/harry-core/src/harry/runner/Query.java
+++ b/harry-core/src/harry/operations/Query.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.operations;
import java.util.ArrayList;
import java.util.Arrays;
@@ -30,9 +30,6 @@ import org.slf4j.LoggerFactory;
import harry.ddl.SchemaSpec;
import harry.model.SelectHelper;
-import harry.operations.CompiledStatement;
-import harry.operations.DeleteHelper;
-import harry.operations.Relation;
import harry.util.Ranges;
import static harry.operations.Relation.FORWARD_COMPARATOR;
diff --git a/harry-core/src/harry/runner/QueryGenerator.java b/harry-core/src/harry/operations/QueryGenerator.java
similarity index 99%
rename from harry-core/src/harry/runner/QueryGenerator.java
rename to harry-core/src/harry/operations/QueryGenerator.java
index 829cf8b..6c815b5 100644
--- a/harry-core/src/harry/runner/QueryGenerator.java
+++ b/harry-core/src/harry/operations/QueryGenerator.java
@@ -16,10 +16,9 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.operations;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.function.LongSupplier;
@@ -34,7 +33,6 @@ import harry.generators.DataGenerators;
import harry.generators.RngUtils;
import harry.generators.Surjections;
import harry.model.OpSelectors;
-import harry.operations.Relation;
// TODO: there's a lot of potential to reduce an amount of garbage here.
// TODO: refactor. Currently, this class is a base for both SELECT and DELETE statements. In retrospect,
diff --git a/harry-core/src/harry/reconciler/Reconciler.java b/harry-core/src/harry/reconciler/Reconciler.java
index a9709b3..da2daa7 100644
--- a/harry-core/src/harry/reconciler/Reconciler.java
+++ b/harry-core/src/harry/reconciler/Reconciler.java
@@ -34,10 +34,10 @@ import harry.core.Run;
import harry.ddl.ColumnSpec;
import harry.ddl.SchemaSpec;
import harry.model.OpSelectors;
-import harry.runner.AbstractPartitionVisitor;
-import harry.runner.PartitionVisitor;
-import harry.runner.Query;
-import harry.runner.QueryGenerator;
+import harry.visitors.AbstractPartitionVisitor;
+import harry.visitors.PartitionVisitor;
+import harry.operations.Query;
+import harry.operations.QueryGenerator;
import harry.util.BitSet;
import harry.util.Ranges;
diff --git a/harry-core/src/harry/runner/Runner.java b/harry-core/src/harry/runner/Runner.java
index 06bc481..b172723 100644
--- a/harry-core/src/harry/runner/Runner.java
+++ b/harry-core/src/harry/runner/Runner.java
@@ -28,7 +28,6 @@ import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -40,6 +39,7 @@ import org.slf4j.LoggerFactory;
import harry.core.Configuration;
import harry.core.Run;
import harry.model.OpSelectors;
+import harry.visitors.PartitionVisitor;
public abstract class Runner
diff --git a/harry-core/src/harry/runner/AbstractPartitionVisitor.java b/harry-core/src/harry/visitors/AbstractPartitionVisitor.java
similarity index 93%
rename from harry-core/src/harry/runner/AbstractPartitionVisitor.java
rename to harry-core/src/harry/visitors/AbstractPartitionVisitor.java
index 0455cbe..3f58e3a 100644
--- a/harry-core/src/harry/runner/AbstractPartitionVisitor.java
+++ b/harry-core/src/harry/visitors/AbstractPartitionVisitor.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import harry.ddl.SchemaSpec;
import harry.model.OpSelectors;
@@ -27,6 +27,11 @@ public abstract class AbstractPartitionVisitor implements PartitionVisitor
protected final OpSelectors.DescriptorSelector descriptorSelector;
protected final SchemaSpec schema;
+ public AbstractPartitionVisitor(AbstractPartitionVisitor visitor)
+ {
+ this(visitor.pdSelector, visitor.descriptorSelector, visitor.schema);
+ }
+
public AbstractPartitionVisitor(OpSelectors.PdSelector pdSelector,
OpSelectors.DescriptorSelector descriptorSelector,
SchemaSpec schema)
diff --git a/harry-core/src/harry/runner/AllPartitionsValidator.java b/harry-core/src/harry/visitors/AllPartitionsValidator.java
similarity index 98%
rename from harry-core/src/harry/runner/AllPartitionsValidator.java
rename to harry-core/src/harry/visitors/AllPartitionsValidator.java
index f42ab65..5a5a1c7 100644
--- a/harry-core/src/harry/runner/AllPartitionsValidator.java
+++ b/harry-core/src/harry/visitors/AllPartitionsValidator.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
@@ -34,6 +34,7 @@ import harry.ddl.SchemaSpec;
import harry.model.Model;
import harry.model.OpSelectors;
import harry.model.sut.SystemUnderTest;
+import harry.operations.Query;
// This might be something that potentially grows into the validator described in the design doc;
// right now it's just a helper/container class
diff --git a/harry-core/src/harry/runner/CorruptingPartitionVisitor.java b/harry-core/src/harry/visitors/CorruptingPartitionVisitor.java
similarity index 97%
rename from harry-core/src/harry/runner/CorruptingPartitionVisitor.java
rename to harry-core/src/harry/visitors/CorruptingPartitionVisitor.java
index d079cf3..71c425b 100644
--- a/harry-core/src/harry/runner/CorruptingPartitionVisitor.java
+++ b/harry-core/src/harry/visitors/CorruptingPartitionVisitor.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import java.util.Random;
import java.util.concurrent.atomic.AtomicLong;
@@ -29,6 +29,8 @@ import harry.corruptor.ChangeValueCorruptor;
import harry.corruptor.HideRowCorruptor;
import harry.corruptor.HideValueCorruptor;
import harry.corruptor.QueryResponseCorruptor;
+import harry.runner.HarryRunner;
+import harry.operations.Query;
public class CorruptingPartitionVisitor implements PartitionVisitor
{
diff --git a/harry-core/src/harry/runner/LoggingPartitionVisitor.java b/harry-core/src/harry/visitors/LoggingPartitionVisitor.java
similarity index 99%
rename from harry-core/src/harry/runner/LoggingPartitionVisitor.java
rename to harry-core/src/harry/visitors/LoggingPartitionVisitor.java
index 9cb827d..3e97ba3 100644
--- a/harry-core/src/harry/runner/LoggingPartitionVisitor.java
+++ b/harry-core/src/harry/visitors/LoggingPartitionVisitor.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import java.io.BufferedWriter;
import java.io.File;
diff --git a/harry-core/src/harry/runner/MutatingPartitionVisitor.java b/harry-core/src/harry/visitors/MutatingPartitionVisitor.java
similarity index 98%
rename from harry-core/src/harry/runner/MutatingPartitionVisitor.java
rename to harry-core/src/harry/visitors/MutatingPartitionVisitor.java
index 4df793e..88456fe 100644
--- a/harry-core/src/harry/runner/MutatingPartitionVisitor.java
+++ b/harry-core/src/harry/visitors/MutatingPartitionVisitor.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import java.util.ArrayList;
import java.util.List;
@@ -29,10 +29,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import harry.core.Run;
-import harry.model.Model;
import harry.model.OpSelectors;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
+import harry.runner.DataTracker;
public class MutatingPartitionVisitor extends AbstractPartitionVisitor
{
diff --git a/harry-core/src/harry/runner/MutatingRowVisitor.java b/harry-core/src/harry/visitors/MutatingRowVisitor.java
similarity index 98%
rename from harry-core/src/harry/runner/MutatingRowVisitor.java
rename to harry-core/src/harry/visitors/MutatingRowVisitor.java
index a14fc96..0c5db9a 100644
--- a/harry-core/src/harry/runner/MutatingRowVisitor.java
+++ b/harry-core/src/harry/visitors/MutatingRowVisitor.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import harry.core.MetricReporter;
import harry.core.Run;
@@ -26,6 +26,8 @@ import harry.model.OpSelectors;
import harry.operations.CompiledStatement;
import harry.operations.DeleteHelper;
import harry.operations.WriteHelper;
+import harry.operations.Query;
+import harry.operations.QueryGenerator;
import harry.util.BitSet;
public class MutatingRowVisitor implements Operation
diff --git a/harry-core/src/harry/runner/Operation.java b/harry-core/src/harry/visitors/Operation.java
similarity index 99%
rename from harry-core/src/harry/runner/Operation.java
rename to harry-core/src/harry/visitors/Operation.java
index f44f3bd..1af21fc 100644
--- a/harry-core/src/harry/runner/Operation.java
+++ b/harry-core/src/harry/visitors/Operation.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import harry.core.Run;
import harry.model.OpSelectors;
diff --git a/harry-core/src/harry/runner/ParallelRecentPartitionValidator.java b/harry-core/src/harry/visitors/ParallelRecentPartitionValidator.java
similarity index 98%
rename from harry-core/src/harry/runner/ParallelRecentPartitionValidator.java
rename to harry-core/src/harry/visitors/ParallelRecentPartitionValidator.java
index b363baf..a1688cd 100644
--- a/harry-core/src/harry/runner/ParallelRecentPartitionValidator.java
+++ b/harry-core/src/harry/visitors/ParallelRecentPartitionValidator.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import java.io.BufferedWriter;
import java.io.File;
@@ -38,6 +38,8 @@ import harry.core.Configuration;
import harry.core.Run;
import harry.generators.Surjections;
import harry.model.Model;
+import harry.operations.Query;
+import harry.operations.QueryGenerator;
public class ParallelRecentPartitionValidator extends ParallelValidator<ParallelRecentPartitionValidator.State>
{
diff --git a/harry-core/src/harry/runner/ParallelValidator.java b/harry-core/src/harry/visitors/ParallelValidator.java
similarity index 99%
rename from harry-core/src/harry/runner/ParallelValidator.java
rename to harry-core/src/harry/visitors/ParallelValidator.java
index 742a7cc..2964eb7 100644
--- a/harry-core/src/harry/runner/ParallelValidator.java
+++ b/harry-core/src/harry/visitors/ParallelValidator.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
diff --git a/harry-core/src/harry/runner/PartitionVisitor.java b/harry-core/src/harry/visitors/PartitionVisitor.java
similarity index 97%
rename from harry-core/src/harry/runner/PartitionVisitor.java
rename to harry-core/src/harry/visitors/PartitionVisitor.java
index 85e3b26..77de711 100644
--- a/harry-core/src/harry/runner/PartitionVisitor.java
+++ b/harry-core/src/harry/visitors/PartitionVisitor.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import harry.core.Run;
diff --git a/harry-core/src/harry/runner/RecentPartitionValidator.java b/harry-core/src/harry/visitors/RecentPartitionValidator.java
similarity index 97%
rename from harry-core/src/harry/runner/RecentPartitionValidator.java
rename to harry-core/src/harry/visitors/RecentPartitionValidator.java
index 69388dd..4bb1e61 100644
--- a/harry-core/src/harry/runner/RecentPartitionValidator.java
+++ b/harry-core/src/harry/visitors/RecentPartitionValidator.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import java.io.BufferedWriter;
import java.io.File;
@@ -24,7 +24,6 @@ import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
-import java.util.Arrays;
import java.util.concurrent.atomic.AtomicLong;
import org.slf4j.Logger;
@@ -35,7 +34,8 @@ import harry.core.Run;
import harry.generators.Surjections;
import harry.model.Model;
import harry.model.OpSelectors;
-import harry.operations.CompiledStatement;
+import harry.operations.Query;
+import harry.operations.QueryGenerator;
public class RecentPartitionValidator implements PartitionVisitor
{
diff --git a/harry-core/src/harry/runner/Sampler.java b/harry-core/src/harry/visitors/Sampler.java
similarity index 99%
rename from harry-core/src/harry/runner/Sampler.java
rename to harry-core/src/harry/visitors/Sampler.java
index c362bd9..e4088d9 100644
--- a/harry-core/src/harry/runner/Sampler.java
+++ b/harry-core/src/harry/visitors/Sampler.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import java.util.concurrent.atomic.AtomicLong;
diff --git a/harry-core/src/harry/runner/SinglePartitionValidator.java b/harry-core/src/harry/visitors/SinglePartitionValidator.java
similarity index 94%
rename from harry-core/src/harry/runner/SinglePartitionValidator.java
rename to harry-core/src/harry/visitors/SinglePartitionValidator.java
index febfc6a..313cff9 100644
--- a/harry-core/src/harry/runner/SinglePartitionValidator.java
+++ b/harry-core/src/harry/visitors/SinglePartitionValidator.java
@@ -16,10 +16,13 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import harry.core.Run;
import harry.model.Model;
+import harry.operations.Query;
+import harry.operations.QueryGenerator;
+import harry.visitors.PartitionVisitor;
public class SinglePartitionValidator implements PartitionVisitor
{
diff --git a/harry-core/test/harry/model/OpSelectorsTest.java b/harry-core/test/harry/model/OpSelectorsTest.java
index 266aeb9..709ba81 100644
--- a/harry-core/test/harry/model/OpSelectorsTest.java
+++ b/harry-core/test/harry/model/OpSelectorsTest.java
@@ -44,9 +44,9 @@ import harry.model.clock.OffsetClock;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
import harry.runner.DataTracker;
-import harry.runner.MutatingPartitionVisitor;
-import harry.runner.PartitionVisitor;
-import harry.runner.Operation;
+import harry.visitors.MutatingPartitionVisitor;
+import harry.visitors.PartitionVisitor;
+import harry.visitors.Operation;
import harry.util.BitSet;
public class OpSelectorsTest
diff --git a/harry-core/test/harry/operations/RelationTest.java b/harry-core/test/harry/operations/RelationTest.java
index 122b224..e0c6e73 100644
--- a/harry-core/test/harry/operations/RelationTest.java
+++ b/harry-core/test/harry/operations/RelationTest.java
@@ -31,8 +31,6 @@ import harry.ddl.ColumnSpec;
import harry.ddl.SchemaSpec;
import harry.generators.DataGeneratorsTest;
import harry.model.OpSelectors;
-import harry.runner.Query;
-import harry.runner.QueryGenerator;
import harry.util.BitSet;
public class RelationTest
diff --git a/harry-integration-external/dependency-reduced-pom.xml b/harry-integration-external/dependency-reduced-pom.xml
new file mode 100644
index 0000000..e2e00a5
--- /dev/null
+++ b/harry-integration-external/dependency-reduced-pom.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <parent>
+ <artifactId>harry-parent</artifactId>
+ <groupId>org.apache.cassandra</groupId>
+ <version>0.0.1-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>harry-integration-external</artifactId>
+ <name>Harry Integration - External</name>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.2.4</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <transformers>
+ <transformer>
+ <mainClass>harry.runner.external.HarryRunnerExternal</mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>hamcrest-core</artifactId>
+ <groupId>org.hamcrest</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.quicktheories</groupId>
+ <artifactId>quicktheories</artifactId>
+ <version>0.25</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/harry-integration/src/harry/runner/FaultInjectingPartitionVisitor.java b/harry-integration/src/harry/runner/FaultInjectingPartitionVisitor.java
index a6483bc..0fba873 100644
--- a/harry-integration/src/harry/runner/FaultInjectingPartitionVisitor.java
+++ b/harry-integration/src/harry/runner/FaultInjectingPartitionVisitor.java
@@ -31,6 +31,9 @@ import harry.model.sut.InJvmSut;
import harry.model.sut.MixedVersionInJvmSut;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
+import harry.visitors.LoggingPartitionVisitor;
+import harry.visitors.Operation;
+import harry.visitors.PartitionVisitor;
public class FaultInjectingPartitionVisitor extends LoggingPartitionVisitor
{
diff --git a/harry-integration/src/harry/runner/QueryingNoOpChecker.java b/harry-integration/src/harry/runner/QueryingNoOpChecker.java
index 08b8e6b..0e7d4fb 100644
--- a/harry-integration/src/harry/runner/QueryingNoOpChecker.java
+++ b/harry-integration/src/harry/runner/QueryingNoOpChecker.java
@@ -25,6 +25,7 @@ import harry.core.Run;
import harry.model.Model;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
+import harry.operations.Query;
public class QueryingNoOpChecker implements Model
{
diff --git a/harry-integration/src/harry/runner/RepairingLocalStateValidator.java b/harry-integration/src/harry/runner/RepairingLocalStateValidator.java
index 98beb93..41926f0 100644
--- a/harry-integration/src/harry/runner/RepairingLocalStateValidator.java
+++ b/harry-integration/src/harry/runner/RepairingLocalStateValidator.java
@@ -32,6 +32,9 @@ import harry.model.QuiescentChecker;
import harry.model.sut.InJvmSut;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
+import harry.operations.Query;
+import harry.visitors.AllPartitionsValidator;
+import harry.visitors.PartitionVisitor;
import static harry.model.SelectHelper.resultSetToRow;
diff --git a/harry-integration/src/harry/runner/Reproduce.java b/harry-integration/src/harry/runner/Reproduce.java
index f3ae9cb..5d34ecd 100644
--- a/harry-integration/src/harry/runner/Reproduce.java
+++ b/harry-integration/src/harry/runner/Reproduce.java
@@ -23,6 +23,7 @@ import java.io.File;
import harry.core.Configuration;
import harry.core.Run;
import harry.model.sut.PrintlnSut;
+import harry.operations.Query;
import harry.reconciler.Reconciler;
public class Reproduce
diff --git a/harry-integration/src/harry/runner/TrivialShrinker.java b/harry-integration/src/harry/runner/TrivialShrinker.java
index 82e0572..23bdbcc 100644
--- a/harry-integration/src/harry/runner/TrivialShrinker.java
+++ b/harry-integration/src/harry/runner/TrivialShrinker.java
@@ -28,6 +28,9 @@ import java.util.function.Predicate;
import harry.core.Configuration;
import harry.core.Run;
+import harry.visitors.AbstractPartitionVisitor;
+import harry.visitors.PartitionVisitor;
+import harry.visitors.SkippingPartitionVisitor;
/**
* A most trivial imaginable shrinker: attempts to skip partitions and/or logical timestamps to see if the
@@ -167,56 +170,6 @@ public class TrivialShrinker
}
}
- public static class SkippingPartitionVisitor extends AbstractPartitionVisitor
- {
- private final AbstractPartitionVisitor delegate;
- private final Set<Long> ltsToSkip;
- private final Set<Long> pdsToSkip;
-
- public SkippingPartitionVisitor(AbstractPartitionVisitor delegate,
- Set<Long> ltsToSkip,
- Set<Long> pdsToSkip)
- {
- super(delegate.pdSelector, delegate.descriptorSelector, delegate.schema);
- this.delegate = delegate;
- this.ltsToSkip = ltsToSkip;
- this.pdsToSkip = pdsToSkip;
- }
-
- protected void beforeLts(long lts, long pd)
- {
- delegate.beforeLts(lts, pd);
- }
-
- protected void afterLts(long lts, long pd)
- {
- delegate.afterLts(lts, pd);
- }
-
- protected void beforeBatch(long lts, long pd, long m)
- {
- delegate.beforeBatch(lts, pd, m);
- }
-
- protected void operation(long lts, long pd, long cd, long m, long opId)
- {
- if (pdsToSkip.contains(pd) || ltsToSkip.contains(lts))
- return;
-
- delegate.operation(lts, pd, cd, m, opId);
- }
-
- protected void afterBatch(long lts, long pd, long m)
- {
- delegate.afterBatch(lts, pd, m);
- }
-
- public void shutdown() throws InterruptedException
- {
- delegate.shutdown();
- }
- }
-
public static String toString(Set<Long> longs)
{
if (longs.isEmpty())
diff --git a/harry-integration/src/harry/visitors/SkippingPartitionVisitor.java b/harry-integration/src/harry/visitors/SkippingPartitionVisitor.java
new file mode 100644
index 0000000..fa910cb
--- /dev/null
+++ b/harry-integration/src/harry/visitors/SkippingPartitionVisitor.java
@@ -0,0 +1,53 @@
+package harry.visitors;
+
+import java.util.Set;
+
+public class SkippingPartitionVisitor extends AbstractPartitionVisitor
+{
+ private final AbstractPartitionVisitor delegate;
+ private final Set<Long> ltsToSkip;
+ private final Set<Long> pdsToSkip;
+
+ public SkippingPartitionVisitor(AbstractPartitionVisitor delegate,
+ Set<Long> ltsToSkip,
+ Set<Long> pdsToSkip)
+ {
+ super(delegate);
+ this.delegate = delegate;
+ this.ltsToSkip = ltsToSkip;
+ this.pdsToSkip = pdsToSkip;
+ }
+
+ protected void beforeLts(long lts, long pd)
+ {
+ delegate.beforeLts(lts, pd);
+ }
+
+ protected void afterLts(long lts, long pd)
+ {
+ delegate.afterLts(lts, pd);
+ }
+
+ protected void beforeBatch(long lts, long pd, long m)
+ {
+ delegate.beforeBatch(lts, pd, m);
+ }
+
+ protected void operation(long lts, long pd, long cd, long m, long opId)
+ {
+ if (pdsToSkip.contains(pd) || ltsToSkip.contains(lts))
+ return;
+
+ delegate.operation(lts, pd, cd, m, opId);
+ }
+
+ protected void afterBatch(long lts, long pd, long m)
+ {
+ delegate.afterBatch(lts, pd, m);
+ }
+
+ public void shutdown() throws InterruptedException
+ {
+ delegate.shutdown();
+ }
+}
diff --git a/harry-integration/test/harry/ddl/SchemaGenTest.java b/harry-integration/test/harry/ddl/SchemaGenTest.java
index 2f7e898..c6204cd 100644
--- a/harry-integration/test/harry/ddl/SchemaGenTest.java
+++ b/harry-integration/test/harry/ddl/SchemaGenTest.java
@@ -32,7 +32,7 @@ import harry.QuickTheoriesAdapter;
import harry.generators.Generator;
import harry.operations.CompiledStatement;
-import harry.runner.Query;
+import harry.operations.Query;
import harry.util.TestRunner;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.Keyspace;
diff --git a/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java b/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
index cadee0c..b819eb5 100644
--- a/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
+++ b/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
@@ -32,10 +32,10 @@ import harry.generators.distribution.Distribution;
import harry.model.NoOpChecker;
import harry.model.OpSelectors;
import harry.model.sut.SystemUnderTest;
-import harry.runner.MutatingPartitionVisitor;
-import harry.runner.MutatingRowVisitor;
-import harry.runner.PartitionVisitor;
-import harry.runner.SinglePartitionValidator;
+import harry.visitors.MutatingPartitionVisitor;
+import harry.visitors.MutatingRowVisitor;
+import harry.visitors.PartitionVisitor;
+import harry.visitors.SinglePartitionValidator;
import harry.util.TestRunner;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.UntypedResultSet;
diff --git a/harry-integration/test/harry/model/ModelTestBase.java b/harry-integration/test/harry/model/ModelTestBase.java
index 5d1659d..70b9fb3 100644
--- a/harry-integration/test/harry/model/ModelTestBase.java
+++ b/harry-integration/test/harry/model/ModelTestBase.java
@@ -28,11 +28,11 @@ import harry.core.Configuration;
import harry.core.Run;
import harry.ddl.SchemaGenerators;
import harry.ddl.SchemaSpec;
-import harry.runner.LoggingPartitionVisitor;
-import harry.runner.MutatingRowVisitor;
-import harry.runner.PartitionVisitor;
+import harry.visitors.LoggingPartitionVisitor;
+import harry.visitors.MutatingRowVisitor;
+import harry.visitors.PartitionVisitor;
import harry.runner.Runner;
-import harry.runner.SinglePartitionValidator;
+import harry.visitors.SinglePartitionValidator;
public abstract class ModelTestBase extends IntegrationTestBase
{
diff --git a/harry-integration/test/harry/model/QuerySelectorNegativeTest.java b/harry-integration/test/harry/model/QuerySelectorNegativeTest.java
index 8c635c8..928e4e9 100644
--- a/harry-integration/test/harry/model/QuerySelectorNegativeTest.java
+++ b/harry-integration/test/harry/model/QuerySelectorNegativeTest.java
@@ -25,6 +25,7 @@ import java.util.Map;
import java.util.Random;
import java.util.function.Supplier;
+import harry.operations.Query;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -39,11 +40,10 @@ import harry.corruptor.HideValueCorruptor;
import harry.corruptor.QueryResponseCorruptor;
import harry.corruptor.ShowValueCorruptor;
import harry.ddl.SchemaGenerators;
-import harry.runner.MutatingPartitionVisitor;
-import harry.runner.MutatingRowVisitor;
-import harry.runner.PartitionVisitor;
-import harry.runner.Query;
-import harry.runner.QueryGenerator;
+import harry.visitors.MutatingPartitionVisitor;
+import harry.visitors.MutatingRowVisitor;
+import harry.visitors.PartitionVisitor;
+import harry.operations.QueryGenerator;
import static harry.corruptor.QueryResponseCorruptor.SimpleQueryResponseCorruptor;
diff --git a/harry-integration/test/harry/model/QuerySelectorTest.java b/harry-integration/test/harry/model/QuerySelectorTest.java
index 484d14a..3436809 100644
--- a/harry-integration/test/harry/model/QuerySelectorTest.java
+++ b/harry-integration/test/harry/model/QuerySelectorTest.java
@@ -31,11 +31,11 @@ import harry.ddl.SchemaGenerators;
import harry.ddl.SchemaSpec;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
-import harry.runner.MutatingPartitionVisitor;
-import harry.runner.MutatingRowVisitor;
-import harry.runner.PartitionVisitor;
-import harry.runner.Query;
-import harry.runner.QueryGenerator;
+import harry.visitors.MutatingPartitionVisitor;
+import harry.visitors.MutatingRowVisitor;
+import harry.visitors.PartitionVisitor;
+import harry.operations.Query;
+import harry.operations.QueryGenerator;
import static harry.generators.DataGenerators.NIL_DESCR;
diff --git a/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java b/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
index 7d5956f..55fb00b 100644
--- a/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
+++ b/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
@@ -29,9 +29,9 @@ import harry.corruptor.HideValueCorruptor;
import harry.corruptor.QueryResponseCorruptor;
import harry.corruptor.QueryResponseCorruptor.SimpleQueryResponseCorruptor;
import harry.ddl.SchemaSpec;
-import harry.runner.PartitionVisitor;
-import harry.runner.Query;
-import harry.runner.SinglePartitionValidator;
+import harry.visitors.PartitionVisitor;
+import harry.operations.Query;
+import harry.visitors.SinglePartitionValidator;
public class QuiescentCheckerIntegrationTest extends ModelTestBase
{
diff --git a/harry-integration/test/harry/model/TestEveryClustering.java b/harry-integration/test/harry/model/TestEveryClustering.java
index f844b41..f3a2ba8 100644
--- a/harry-integration/test/harry/model/TestEveryClustering.java
+++ b/harry-integration/test/harry/model/TestEveryClustering.java
@@ -1,28 +1,22 @@
package harry.model;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.function.Supplier;
-
-import org.junit.Assert;
-import org.junit.Test;
-
import harry.core.Configuration;
import harry.core.Run;
import harry.ddl.SchemaGenerators;
import harry.ddl.SchemaSpec;
import harry.generators.distribution.Distribution;
-import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
+import harry.operations.Query;
import harry.operations.Relation;
-import harry.runner.FaultInjectingPartitionVisitor;
-import harry.runner.LoggingPartitionVisitor;
-import harry.runner.MutatingPartitionVisitor;
-import harry.runner.MutatingRowVisitor;
-import harry.runner.PartitionVisitor;
-import harry.runner.Query;
-import harry.runner.QueryGenerator;
+import harry.visitors.LoggingPartitionVisitor;
+import harry.visitors.MutatingRowVisitor;
+import harry.visitors.PartitionVisitor;
import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.function.Supplier;
public class TestEveryClustering extends IntegrationTestBase
{
diff --git a/harry-integration/test/harry/op/RowVisitorTest.java b/harry-integration/test/harry/op/RowVisitorTest.java
index ae51115..a644afa 100644
--- a/harry-integration/test/harry/op/RowVisitorTest.java
+++ b/harry-integration/test/harry/op/RowVisitorTest.java
@@ -35,7 +35,7 @@ import harry.model.clock.OffsetClock;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
import harry.runner.DataTracker;
-import harry.runner.MutatingRowVisitor;
+import harry.visitors.MutatingRowVisitor;
import org.apache.cassandra.cql3.CQLTester;
import static harry.model.OpSelectors.DefaultDescriptorSelector.DEFAULT_OP_SELECTOR;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra-harry] 01/04: Core improvements
Posted by if...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch CASSANDRA-16262-2
in repository https://gitbox.apache.org/repos/asf/cassandra-harry.git
commit f22d52e8e995411fd8190c8d282582975a8e02b1
Author: Alex Petrov <ol...@gmail.com>
AuthorDate: Mon Jul 12 17:04:37 2021 +0200
Core improvements
Major features:
* Implement updates
* Make sure we can advance RNGs from zero as well
* Fix a problem with predictable descriptor
Bugfixes:
* Fix column mask inconsistencies
* Fix a problem with partition key liveness info
Quality of life improvements:
* Get rid of driver dependency for query generation
* Get rid of guava dependency
* Add reusable config files
* Switch from streams to iterables
General improvements:
* Make unset and nil descriptors more distinct and harder to generate particularly for the smaller descriptors
* Fixed schema configurator to allow empty column sets
* Move workloads to a common dir
* Fixed schema configurator to output correct json
* No-op checker to execute with Quorum, not ALL
* Make tag for build unique
Patch by Alex Petrov for CASSANDRA-16262
---
harry-core/pom.xml | 9 +-
harry-core/src/harry/core/Configuration.java | 44 +++++-
harry-core/src/harry/core/VisibleForTesting.java | 5 +
harry-core/src/harry/corruptor/RowCorruptor.java | 2 +-
harry-core/src/harry/ddl/ColumnSpec.java | 10 +-
harry-core/src/harry/ddl/SchemaGenerators.java | 76 +++++----
harry-core/src/harry/ddl/SchemaSpec.java | 102 ++++++++++---
.../src/harry/generators/DataGenerators.java | 17 ++-
.../src/harry/generators/RandomGenerator.java | 3 +-
harry-core/src/harry/generators/RngUtils.java | 6 +-
harry-core/src/harry/generators/Surjections.java | 2 +-
.../harry/model/AlwaysSamePartitionSelector.java | 69 +++++++++
harry-core/src/harry/model/NoOpChecker.java | 3 +-
harry-core/src/harry/model/OpSelectors.java | 54 +++++--
harry-core/src/harry/model/SelectHelper.java | 114 ++++++++++----
.../model/clock/ApproximateMonotonicClock.java | 3 +-
harry-core/src/harry/model/clock/OffsetClock.java | 20 +++
harry-core/src/harry/model/sut/PrintlnSut.java | 18 +++
harry-core/src/harry/operations/DeleteHelper.java | 61 ++++----
harry-core/src/harry/operations/Relation.java | 80 ++--------
harry-core/src/harry/operations/WriteHelper.java | 170 +++++++++++----------
harry-core/src/harry/reconciler/Reconciler.java | 61 +++++---
.../harry/runner/CorruptingPartitionVisitor.java | 1 -
harry-core/src/harry/runner/DataTracker.java | 3 +
.../src/harry/runner/DefaultDataTracker.java | 2 +-
.../src/harry/runner/MutatingPartitionVisitor.java | 1 +
.../src/harry/runner/MutatingRowVisitor.java | 44 +++++-
harry-core/src/harry/runner/Operation.java | 20 ++-
harry-core/src/harry/runner/QueryGenerator.java | 2 +-
harry-core/src/harry/util/BitSet.java | 2 +-
harry-core/src/harry/util/TestRunner.java | 9 +-
harry-core/test/harry/model/OpSelectorsTest.java | 34 +++--
32 files changed, 691 insertions(+), 356 deletions(-)
diff --git a/harry-core/pom.xml b/harry-core/pom.xml
index fef070d..d313e7a 100755
--- a/harry-core/pom.xml
+++ b/harry-core/pom.xml
@@ -33,10 +33,11 @@
<name>Harry Core</name>
<dependencies>
- <dependency>
- <groupId>com.datastax.cassandra</groupId>
- <artifactId>cassandra-driver-core</artifactId>
- </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.25</version>
+ </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
diff --git a/harry-core/src/harry/core/Configuration.java b/harry-core/src/harry/core/Configuration.java
index bd011c2..4a2a1e6 100644
--- a/harry-core/src/harry/core/Configuration.java
+++ b/harry-core/src/harry/core/Configuration.java
@@ -38,10 +38,13 @@ import harry.ddl.SchemaGenerators;
import harry.ddl.SchemaSpec;
import harry.generators.Surjections;
import harry.generators.distribution.Distribution;
+import harry.model.AlwaysSamePartitionSelector;
import harry.model.Model;
import harry.model.OpSelectors;
import harry.model.QuiescentChecker;
import harry.model.clock.ApproximateMonotonicClock;
+import harry.model.clock.OffsetClock;
+import harry.model.sut.PrintlnSut;
import harry.model.sut.SystemUnderTest;
import harry.runner.AllPartitionsValidator;
import harry.runner.CorruptingPartitionVisitor;
@@ -92,6 +95,11 @@ public class Configuration
mapper.registerSubtypes(CorruptingPartitionVisitorConfiguration.class);
mapper.registerSubtypes(RecentPartitionsValidatorConfiguration.class);
mapper.registerSubtypes(FixedSchemaProviderConfiguration.class);
+ mapper.registerSubtypes(AlwaysSamePartitionSelector.AlwaysSamePartitionSelectorConfiguration.class);
+ mapper.registerSubtypes(OffsetClock.OffsetClockConfiguration.class);
+ mapper.registerSubtypes(PrintlnSut.PrintlnSutConfiguration.class);
+ mapper.registerSubtypes(NoOpDataTrackerConfiguration.class);
+ mapper.registerSubtypes(NoOpMetricReporterConfiguration.class);
}
public final long seed;
@@ -407,7 +415,7 @@ public class Configuration
}
- @JsonTypeName("no_op_tracker")
+ @JsonTypeName("no_op")
public static class NoOpDataTrackerConfiguration implements DataTrackerConfiguration
{
@JsonCreator
@@ -613,7 +621,7 @@ public class Configuration
}
}
- @JsonTypeName("no_op_checker")
+ @JsonTypeName("no_op")
public static class NoOpCheckerConfig implements ModelConfiguration
{
@JsonCreator
@@ -692,7 +700,7 @@ public class Configuration
private Map<OpSelectors.OperationKind, Integer> operation_kind_weights = new OperationKindSelectorBuilder()
.addWeight(OpSelectors.OperationKind.DELETE_ROW, 1)
.addWeight(OpSelectors.OperationKind.DELETE_COLUMN, 1)
- .addWeight(OpSelectors.OperationKind.WRITE, 98)
+ .addWeight(OpSelectors.OperationKind.INSERT, 98)
.build();
private Map<OpSelectors.OperationKind, long[]> column_mask_bitsets;
private int[] fractions;
@@ -1057,6 +1065,12 @@ public class Configuration
@JsonTypeName("fixed")
public static class FixedSchemaProviderConfiguration implements SchemaProviderConfiguration
{
+ public final String keyspace;
+ public final String table;
+ public final Map<String, String> partition_keys;
+ public final Map<String, String> clustering_keys;
+ public final Map<String, String> regular_columns;
+ public final Map<String, String> static_keys;
private final SchemaSpec schemaSpec;
@JsonCreator
@@ -1067,10 +1081,28 @@ public class Configuration
@JsonProperty("regular_columns") Map<String, String> regulars,
@JsonProperty("static_columns") Map<String, String> statics)
{
- this.schemaSpec = SchemaGenerators.parse(keyspace, table,
- pks, cks, regulars, statics);
+ this(SchemaGenerators.parse(keyspace, table,
+ pks, cks, regulars, statics),
+ pks,
+ cks,
+ regulars,
+ statics);
}
+ public FixedSchemaProviderConfiguration(SchemaSpec schemaSpec,
+ Map<String, String> pks,
+ Map<String, String> cks,
+ Map<String, String> regulars,
+ Map<String, String> statics)
+ {
+ this.schemaSpec = schemaSpec;
+ this.keyspace = schemaSpec.keyspace;
+ this.table = schemaSpec.table;
+ this.partition_keys = pks;
+ this.clustering_keys = cks;
+ this.regular_columns = regulars;
+ this.static_keys = statics;
+ }
public SchemaSpec make(long seed)
{
return schemaSpec;
@@ -1082,7 +1114,7 @@ public class Configuration
{
}
- @JsonTypeName("default")
+ @JsonTypeName("no_op")
public static class NoOpMetricReporterConfiguration implements MetricReporterConfiguration
{
public MetricReporter make()
diff --git a/harry-core/src/harry/core/VisibleForTesting.java b/harry-core/src/harry/core/VisibleForTesting.java
new file mode 100644
index 0000000..efa712e
--- /dev/null
+++ b/harry-core/src/harry/core/VisibleForTesting.java
@@ -0,0 +1,5 @@
+package harry.core;
+
+public @interface VisibleForTesting {
+}
+
diff --git a/harry-core/src/harry/corruptor/RowCorruptor.java b/harry-core/src/harry/corruptor/RowCorruptor.java
index 4c6b005..7b19cf1 100644
--- a/harry-core/src/harry/corruptor/RowCorruptor.java
+++ b/harry-core/src/harry/corruptor/RowCorruptor.java
@@ -29,7 +29,7 @@ import harry.operations.CompiledStatement;
public interface RowCorruptor
{
- Logger logger = LoggerFactory.getLogger(QueryResponseCorruptor.class);
+ final Logger logger = LoggerFactory.getLogger(QueryResponseCorruptor.class);
boolean canCorrupt(ResultSetRow row);
diff --git a/harry-core/src/harry/ddl/ColumnSpec.java b/harry-core/src/harry/ddl/ColumnSpec.java
index 94e9881..6d652c4 100644
--- a/harry-core/src/harry/ddl/ColumnSpec.java
+++ b/harry-core/src/harry/ddl/ColumnSpec.java
@@ -18,20 +18,18 @@
package harry.ddl;
+import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.UUID;
-import com.google.common.collect.ImmutableList;
-
import harry.generators.Bijections;
import harry.generators.StringBijection;
-import static harry.generators.StringBijection.getByte;
-
public class ColumnSpec<T>
{
public final String name;
@@ -318,7 +316,7 @@ public class ColumnSpec<T>
}
};
- public static final Collection<DataType<?>> DATA_TYPES = ImmutableList.of(
+ public static final Collection<DataType<?>> DATA_TYPES = Collections.unmodifiableList(Arrays.asList(
ColumnSpec.int8Type,
ColumnSpec.int16Type,
ColumnSpec.int32Type,
@@ -328,7 +326,7 @@ public class ColumnSpec<T>
ColumnSpec.doubleType,
ColumnSpec.asciiType,
ColumnSpec.uuidType,
- ColumnSpec.timestampType);
+ ColumnSpec.timestampType));
public static class ReversedType<T> extends DataType<T>
{
diff --git a/harry-core/src/harry/ddl/SchemaGenerators.java b/harry-core/src/harry/ddl/SchemaGenerators.java
index 47936e1..8508d44 100644
--- a/harry-core/src/harry/ddl/SchemaGenerators.java
+++ b/harry-core/src/harry/ddl/SchemaGenerators.java
@@ -21,16 +21,14 @@ package harry.ddl;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.function.Supplier;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
import harry.generators.Generator;
import harry.generators.Surjections;
@@ -43,32 +41,30 @@ public class SchemaGenerators
return new Builder(ks);
}
- public static final Collection<ColumnSpec.DataType<?>> clusteringKeyTypes;
public static final Map<String, ColumnSpec.DataType<?>> nameToTypeMap;
public static final Collection<ColumnSpec.DataType<?>> columnTypes;
+ public static final Collection<ColumnSpec.DataType<?>> partitionKeyTypes;
+ public static final Collection<ColumnSpec.DataType<?>> clusteringKeyTypes;
static
{
+ partitionKeyTypes = Collections.unmodifiableList(Arrays.asList(ColumnSpec.int64Type,
+ ColumnSpec.asciiType,
+ ColumnSpec.asciiType(4, 5),
+ ColumnSpec.asciiType(4, 10)));
+
+ columnTypes = Collections.unmodifiableList(Arrays.asList(
+// ColumnSpec.int8Type,
+// ColumnSpec.int16Type,
+// ColumnSpec.int32Type,
+ ColumnSpec.int64Type,
+ ColumnSpec.asciiType,
+ ColumnSpec.asciiType(4, 256),
+ ColumnSpec.asciiType(4, 512)));
+
- ImmutableList.Builder<ColumnSpec.DataType<?>> builder = ImmutableList.builder();
- builder.add(ColumnSpec.int8Type,
- ColumnSpec.int16Type,
- ColumnSpec.int32Type,
- ColumnSpec.int64Type,
- ColumnSpec.asciiType,
- ColumnSpec.asciiType(4, 256),
- ColumnSpec.asciiType(4, 512));
-
- columnTypes = builder.build();
- builder.add(ColumnSpec.int8Type,
- ColumnSpec.int16Type,
- ColumnSpec.int32Type,
- ColumnSpec.int64Type,
- ColumnSpec.asciiType);
- builder = ImmutableList.builder();
- builder.addAll(columnTypes);
-
- ImmutableMap.Builder<String, ColumnSpec.DataType<?>> mapBuilder = ImmutableMap.builder();
+ List<ColumnSpec.DataType<?>> builder = new ArrayList<>(columnTypes);
+ Map<String, ColumnSpec.DataType<?>> mapBuilder = new HashMap<>();
for (ColumnSpec.DataType<?> columnType : columnTypes)
{
@@ -82,8 +78,8 @@ public class SchemaGenerators
builder.add(ColumnSpec.floatType);
builder.add(ColumnSpec.doubleType);
- clusteringKeyTypes = builder.build();
- nameToTypeMap = mapBuilder.build();
+ clusteringKeyTypes = Collections.unmodifiableList(builder);
+ nameToTypeMap = Collections.unmodifiableMap(mapBuilder);
}
@SuppressWarnings("unchecked")
@@ -126,7 +122,7 @@ public class SchemaGenerators
public ColumnSpec<?> apply(ColumnSpec.DataType<?> type)
{
- return new ColumnSpec<>(prefix + (counter++),
+ return new ColumnSpec<>(String.format("%s%04d", prefix, counter++),
type,
kind);
}
@@ -143,7 +139,24 @@ public class SchemaGenerators
public ColumnSpec<?> apply(ColumnSpec.DataType<?> type)
{
- return ColumnSpec.ck(prefix + (counter++), type);
+ return ColumnSpec.ck(String.format("%s%04d", prefix, counter++), type);
+ }
+ });
+ }
+
+ @SuppressWarnings("unchecked")
+ public static Generator<ColumnSpec<?>> partitionColumnSpecGenerator(String prefix)
+ {
+ return fromValues(partitionKeyTypes)
+ .map(new Function<ColumnSpec.DataType<?>, ColumnSpec<?>>()
+ {
+ private int counter = 0;
+
+ public ColumnSpec<?> apply(ColumnSpec.DataType<?> type)
+ {
+
+ return ColumnSpec.pk(String.format("%s%04d", prefix, counter++),
+ type);
}
});
}
@@ -155,10 +168,10 @@ public class SchemaGenerators
private final String keyspace;
private final Supplier<String> tableNameSupplier;
- private Generator<ColumnSpec<?>> pkGenerator = columnSpecGenerator("pk", ColumnSpec.Kind.PARTITION_KEY);
+ private Generator<ColumnSpec<?>> pkGenerator = partitionColumnSpecGenerator("pk");
private Generator<ColumnSpec<?>> ckGenerator = clusteringColumnSpecGenerator("ck");
private Generator<ColumnSpec<?>> regularGenerator = columnSpecGenerator("regular", ColumnSpec.Kind.REGULAR);
- private Generator<ColumnSpec<?>> staticGenerator = columnSpecGenerator("regular", ColumnSpec.Kind.STATIC);
+ private Generator<ColumnSpec<?>> staticGenerator = columnSpecGenerator("static", ColumnSpec.Kind.STATIC);
private int minPks = 1;
private int maxPks = 1;
@@ -457,6 +470,9 @@ public class SchemaGenerators
public static List<ColumnSpec<?>> toColumns(Map<String, String> config, ColumnSpec.Kind kind, boolean allowReverse)
{
+ if (config == null)
+ return Collections.EMPTY_LIST;
+
List<ColumnSpec<?>> columns = new ArrayList<>(config.size());
for (Map.Entry<String, String> e : config.entrySet())
diff --git a/harry-core/src/harry/ddl/SchemaSpec.java b/harry-core/src/harry/ddl/SchemaSpec.java
index f07c106..eba87c6 100644
--- a/harry-core/src/harry/ddl/SchemaSpec.java
+++ b/harry-core/src/harry/ddl/SchemaSpec.java
@@ -18,14 +18,12 @@
package harry.ddl;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.function.Consumer;
-import java.util.stream.Stream;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Streams;
import harry.generators.DataGenerators;
import harry.operations.CompiledStatement;
@@ -87,22 +85,30 @@ public class SchemaSpec
this.keyspace = keyspace;
this.table = table;
this.isCompactStorage = isCompactStorage;
- this.partitionKeys = ImmutableList.copyOf(partitionKeys);
+
+ this.partitionKeys = Collections.unmodifiableList(new ArrayList<>(partitionKeys));
for (int i = 0; i < partitionKeys.size(); i++)
partitionKeys.get(i).setColumnIndex(i);
- this.clusteringKeys = ImmutableList.copyOf(clusteringKeys);
+ this.clusteringKeys = Collections.unmodifiableList(new ArrayList<>(clusteringKeys));
for (int i = 0; i < clusteringKeys.size(); i++)
clusteringKeys.get(i).setColumnIndex(i);
- this.staticColumns = ImmutableList.copyOf(staticColumns);
+ this.staticColumns = Collections.unmodifiableList(new ArrayList<>(staticColumns));
for (int i = 0; i < staticColumns.size(); i++)
staticColumns.get(i).setColumnIndex(i);
- this.regularColumns = ImmutableList.copyOf(regularColumns);
+ this.regularColumns = Collections.unmodifiableList(new ArrayList<>(regularColumns));
for (int i = 0; i < regularColumns.size(); i++)
regularColumns.get(i).setColumnIndex(i);
- this.allColumns = ImmutableList.copyOf(Iterables.concat(partitionKeys,
- clusteringKeys,
- staticColumns,
- regularColumns));
+
+ List<ColumnSpec<?>> all = new ArrayList<>();
+ for (ColumnSpec<?> columnSpec : concat(partitionKeys,
+ clusteringKeys,
+ staticColumns,
+ regularColumns))
+ {
+ all.add(columnSpec);
+ }
+ this.allColumns = Collections.unmodifiableList(all);
+
this.pkGenerator = DataGenerators.createKeyGenerator(partitionKeys);
this.ckGenerator = DataGenerators.createKeyGenerator(clusteringKeys);
@@ -122,7 +128,6 @@ public class SchemaSpec
}
// todo: bitset views?
-
public BitSet regularColumnsMask()
{
return this.regularColumnsMask;
@@ -254,13 +259,13 @@ public class SchemaSpec
sb.append(" PRIMARY KEY");
}
- Streams.concat(clusteringKeys.stream(),
- staticColumns.stream(),
- regularColumns.stream())
- .forEach((cd) -> {
- commaAppender.accept(sb);
- sb.append(cd.toCQL());
- });
+ for (ColumnSpec<?> cd : concat(clusteringKeys,
+ staticColumns,
+ regularColumns))
+ {
+ commaAppender.accept(sb);
+ sb.append(cd.toCQL());
+ }
if (clusteringKeys.size() > 0 || partitionKeys.size() > 1)
{
@@ -409,4 +414,59 @@ public class SchemaSpec
{
return Objects.hash(keyspace, table, partitionKeys, clusteringKeys, regularColumns);
}
+
+ public static <T> Iterable<T> concat(Iterable<T>... iterables)
+ {
+ assert iterables != null && iterables.length > 0;
+ if (iterables.length == 1)
+ return iterables[0];
+
+ return () -> {
+ return new Iterator<T>()
+ {
+ int idx;
+ Iterator<T> current;
+ boolean hasNext;
+
+ {
+ idx = 0;
+ prepareNext();
+ }
+
+ private void prepareNext()
+ {
+ if (current != null && current.hasNext())
+ {
+ hasNext = true;
+ return;
+ }
+
+ while (idx < iterables.length)
+ {
+ current = iterables[idx].iterator();
+ idx++;
+ if (current.hasNext())
+ {
+ hasNext = true;
+ return;
+ }
+ }
+
+ hasNext = false;
+ }
+
+ public boolean hasNext()
+ {
+ return hasNext;
+ }
+
+ public T next()
+ {
+ T next = current.next();
+ prepareNext();
+ return next;
+ }
+ };
+ };
+ }
}
diff --git a/harry-core/src/harry/generators/DataGenerators.java b/harry-core/src/harry/generators/DataGenerators.java
index 6878ae4..e87e7dc 100644
--- a/harry-core/src/harry/generators/DataGenerators.java
+++ b/harry-core/src/harry/generators/DataGenerators.java
@@ -22,15 +22,22 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import com.google.common.annotations.VisibleForTesting;
-
+import harry.core.VisibleForTesting;
import harry.ddl.ColumnSpec;
public class DataGenerators
{
- public static final Object UNSET_VALUE = new Object();
- public static long UNSET_DESCR = 0;
- public static long NIL_DESCR = -1;
+ public static final Object UNSET_VALUE = new Object() {
+ public String toString()
+ {
+ return "UNSET";
+ }
+ };
+
+ // There is still a slim chance that we're going to produce either of these values by chance, but we'll catch this
+ // during value generation
+ public static long UNSET_DESCR = Long.MAX_VALUE;
+ public static long NIL_DESCR = Long.MIN_VALUE;
public static Object[] inflateData(List<ColumnSpec<?>> columns, long[] descriptors)
{
diff --git a/harry-core/src/harry/generators/RandomGenerator.java b/harry-core/src/harry/generators/RandomGenerator.java
index 869f60e..a1ca125 100644
--- a/harry-core/src/harry/generators/RandomGenerator.java
+++ b/harry-core/src/harry/generators/RandomGenerator.java
@@ -18,8 +18,7 @@
package harry.generators;
-import com.google.common.annotations.VisibleForTesting;
-
+import harry.core.VisibleForTesting;
/**
* Random generator interface that offers:
diff --git a/harry-core/src/harry/generators/RngUtils.java b/harry-core/src/harry/generators/RngUtils.java
index 749cf7f..894204f 100644
--- a/harry-core/src/harry/generators/RngUtils.java
+++ b/harry-core/src/harry/generators/RngUtils.java
@@ -22,8 +22,12 @@ import java.util.function.LongSupplier;
public class RngUtils
{
+ private static final long CONSTANT = 0x2545F4914F6CDD1DL;
public static long next(long input)
{
+ if (input == 0)
+ return next(CONSTANT);
+
return xorshift64star(input);
}
@@ -32,7 +36,7 @@ public class RngUtils
input ^= input >> 12;
input ^= input << 25; // b
input ^= input >> 27; // c
- return input * 0x2545F4914F6CDD1DL;
+ return input * CONSTANT;
}
public static long[] next(long current, int n)
diff --git a/harry-core/src/harry/generators/Surjections.java b/harry-core/src/harry/generators/Surjections.java
index e5a937c..13f66ff 100644
--- a/harry-core/src/harry/generators/Surjections.java
+++ b/harry-core/src/harry/generators/Surjections.java
@@ -26,7 +26,7 @@ import java.util.function.Function;
import java.util.function.LongFunction;
import java.util.function.Supplier;
-import com.google.common.annotations.VisibleForTesting;
+import harry.core.VisibleForTesting;
public class Surjections
{
diff --git a/harry-core/src/harry/model/AlwaysSamePartitionSelector.java b/harry-core/src/harry/model/AlwaysSamePartitionSelector.java
new file mode 100644
index 0000000..43160fb
--- /dev/null
+++ b/harry-core/src/harry/model/AlwaysSamePartitionSelector.java
@@ -0,0 +1,69 @@
+package harry.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import harry.core.Configuration;
+
+/**
+ * A simple test-only descriptor selector that can used for testing things where you only need one partition
+ */
+public class AlwaysSamePartitionSelector extends OpSelectors.PdSelector
+{
+ private final long pd;
+
+ public AlwaysSamePartitionSelector(long pd)
+ {
+ this.pd = pd;
+ }
+
+ protected long pd(long lts)
+ {
+ return 0;
+ }
+
+ public long nextLts(long lts)
+ {
+ return lts + 1;
+ }
+
+ public long prevLts(long lts)
+ {
+ return lts - 1;
+ }
+
+ public long maxLtsFor(long pd)
+ {
+ return 1000;
+ }
+
+ public long minLtsAt(long position)
+ {
+ return 0;
+ }
+
+ public long minLtsFor(long pd)
+ {
+ return 0;
+ }
+
+ public long positionFor(long lts)
+ {
+ return 0;
+ }
+
+ @JsonTypeName("always_same")
+ public static class AlwaysSamePartitionSelectorConfiguration implements Configuration.PDSelectorConfiguration
+ {
+ private final long pd;
+
+ public AlwaysSamePartitionSelectorConfiguration(@JsonProperty("pd") long pd)
+ {
+ this.pd = pd;
+ }
+
+ public OpSelectors.PdSelector make(OpSelectors.Rng rng)
+ {
+ return new AlwaysSamePartitionSelector(pd);
+ }
+ }
+}
diff --git a/harry-core/src/harry/model/NoOpChecker.java b/harry-core/src/harry/model/NoOpChecker.java
index 4cf4606..a13b6ec 100644
--- a/harry-core/src/harry/model/NoOpChecker.java
+++ b/harry-core/src/harry/model/NoOpChecker.java
@@ -34,6 +34,7 @@ public class NoOpChecker implements Model
public void validate(Query query)
{
run.sut.execute(query.toSelectStatement(),
- SystemUnderTest.ConsistencyLevel.ALL);
+ // TODO: make it configurable
+ SystemUnderTest.ConsistencyLevel.QUORUM);
}
}
diff --git a/harry-core/src/harry/model/OpSelectors.java b/harry-core/src/harry/model/OpSelectors.java
index 3adbb95..3dd4a25 100644
--- a/harry-core/src/harry/model/OpSelectors.java
+++ b/harry-core/src/harry/model/OpSelectors.java
@@ -22,9 +22,8 @@ import java.util.EnumMap;
import java.util.List;
import java.util.Map;
-import com.google.common.annotations.VisibleForTesting;
-
import harry.core.Configuration;
+import harry.core.VisibleForTesting;
import harry.ddl.ColumnSpec;
import harry.ddl.SchemaSpec;
import harry.generators.Bytes;
@@ -34,6 +33,7 @@ import harry.generators.Surjections;
import harry.generators.distribution.Distribution;
import harry.util.BitSet;
+import static harry.generators.DataGenerators.NIL_DESCR;
import static harry.generators.DataGenerators.UNSET_DESCR;
/**
@@ -179,18 +179,20 @@ public interface OpSelectors
public long[] vds(long pd, long cd, long lts, long opId, SchemaSpec schema)
{
- return descriptors(pd, cd, lts, opId, schema.regularColumns, schema.regularColumnsMask(), schema.regularColumnsOffset);
+ BitSet setColumns = columnMask(pd, lts, opId);
+ return descriptors(pd, cd, lts, opId, schema.regularColumns, schema.regularColumnsMask(), setColumns, schema.regularColumnsOffset);
}
public long[] sds(long pd, long cd, long lts, long opId, SchemaSpec schema)
{
- return descriptors(pd, cd, lts, opId, schema.staticColumns, schema.staticColumnsMask(), schema.staticColumnsOffset);
+ BitSet setColumns = columnMask(pd, lts, opId);
+ return descriptors(pd, cd, lts, opId, schema.staticColumns, schema.staticColumnsMask(), setColumns, schema.staticColumnsOffset);
}
- public long[] descriptors(long pd, long cd, long lts, long opId, List<ColumnSpec<?>> columns, BitSet mask, int offset)
+ private long[] descriptors(long pd, long cd, long lts, long opId, List<ColumnSpec<?>> columns, BitSet mask, BitSet setColumns, int offset)
{
+ assert opId < opsPerModification(lts) * numberOfModifications(lts) : String.format("Operation id %d exceeds the maximum expected number of operations %d", opId, opsPerModification(lts) * numberOfModifications(lts));
long[] descriptors = new long[columns.size()];
- BitSet setColumns = columnMask(pd, cd, opId);
for (int i = 0; i < descriptors.length; i++)
{
@@ -199,6 +201,9 @@ public interface OpSelectors
{
ColumnSpec<?> spec = columns.get(i);
long vd = vd(pd, cd, lts, opId, col) & Bytes.bytePatternFor(spec.type.maxSize());
+ assert vd != UNSET_DESCR : "Ambiguous unset descriptor generated for the value";
+ assert vd != NIL_DESCR : "Ambiguous nil descriptor generated for the value";
+
descriptors[i] = vd;
}
else
@@ -387,13 +392,13 @@ public interface OpSelectors
switch (type)
{
+ case UPDATE_WITH_STATICS:
case DELETE_COLUMN_WITH_STATICS:
gen = (descriptor) -> {
long counter = 0;
while (counter <= 100)
{
BitSet bitSet = orig.inflate(descriptor);
-
if ((schema.regularColumns.isEmpty() || !bitSet.allUnset(schema.regularColumnsMask))
&& (schema.staticColumns.isEmpty() || !bitSet.allUnset(schema.staticColumnsMask)))
return bitSet;
@@ -404,6 +409,23 @@ public interface OpSelectors
throw new RuntimeException(String.format("Could not generate a value after %d attempts.", counter));
};
break;
+ // Can not have an UPDATE statement without anything to update
+ case UPDATE:
+ gen = descriptor -> {
+ long counter = 0;
+ while (counter <= 100)
+ {
+ BitSet bitSet = orig.inflate(descriptor);
+
+ if (!bitSet.allUnset(schema.regularColumnsMask))
+ return bitSet;
+
+ descriptor = RngUtils.next(descriptor);
+ counter++;
+ }
+ throw new RuntimeException(String.format("Could not generate a value after %d attempts.", counter));
+ };
+ break;
case DELETE_COLUMN:
gen = (descriptor) -> {
long counter = 0;
@@ -429,7 +451,7 @@ public interface OpSelectors
public ColumnSelectorBuilder forWrite(Surjections.Surjection<BitSet> gen)
{
- m.put(OperationKind.WRITE, gen);
+ m.put(OperationKind.INSERT, gen);
return this;
}
@@ -547,8 +569,8 @@ public interface OpSelectors
protected final static long BITSET_IDX_STREAM = 0x92eb607bef1L;
public static OperationSelector DEFAULT_OP_SELECTOR = OperationSelector.weighted(Surjections.weights(45, 45, 3, 2, 2, 1, 1, 1),
- OperationKind.WRITE,
- OperationKind.WRITE_WITH_STATICS,
+ OperationKind.INSERT,
+ OperationKind.INSERT_WITH_STATICS,
OperationKind.DELETE_ROW,
OperationKind.DELETE_COLUMN,
OperationKind.DELETE_COLUMN_WITH_STATICS,
@@ -649,7 +671,8 @@ public interface OpSelectors
public OperationKind operationType(long pd, long lts, long opId)
{
- return operationType(pd, lts, opId, partitionLevelOperationsMask(pd, lts));
+ OperationKind kind = operationType(pd, lts, opId, partitionLevelOperationsMask(pd, lts));
+ return kind;
}
// TODO: create this bitset once per lts
@@ -666,7 +689,8 @@ public interface OpSelectors
public OperationKind operationType(long pd, long lts, long opId, BitSet partitionLevelOperationsMask)
{
- return operationSelector.inflate(pd ^ lts ^ opId, partitionLevelOperationsMask.isSet((int) opId));
+ long descriptor = rng.randomNumber(pd ^ lts ^ opId, BITSET_IDX_STREAM);
+ return operationSelector.inflate(descriptor, partitionLevelOperationsMask.isSet((int) opId));
}
public BitSet columnMask(long pd, long lts, long opId)
@@ -688,8 +712,10 @@ public interface OpSelectors
public enum OperationKind
{
- WRITE(false),
- WRITE_WITH_STATICS(true),
+ UPDATE(false),
+ INSERT(false),
+ UPDATE_WITH_STATICS(true),
+ INSERT_WITH_STATICS(true),
DELETE_PARTITION(true),
DELETE_ROW(false),
DELETE_COLUMN(false),
diff --git a/harry-core/src/harry/model/SelectHelper.java b/harry-core/src/harry/model/SelectHelper.java
index 70d1eb2..fc8f6f7 100644
--- a/harry-core/src/harry/model/SelectHelper.java
+++ b/harry-core/src/harry/model/SelectHelper.java
@@ -19,13 +19,9 @@
package harry.model;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import com.datastax.driver.core.querybuilder.Ordering;
-import com.datastax.driver.core.querybuilder.QueryBuilder;
-import com.datastax.driver.core.querybuilder.Select;
import harry.data.ResultSetRow;
import harry.ddl.ColumnSpec;
import harry.ddl.SchemaSpec;
@@ -48,67 +44,119 @@ public class SelectHelper
*/
public static CompiledStatement select(SchemaSpec schema, long pd, List<Relation> relations, boolean reverse, boolean includeWriteTime)
{
- Select.Selection select = QueryBuilder.select();
- for (ColumnSpec<?> column : schema.allColumns)
- select.column(column.name);
+ StringBuilder b = new StringBuilder();
+ b.append("SELECT ");
+
+ for (int i = 0; i < schema.allColumns.size(); i++)
+ {
+ ColumnSpec<?> spec = schema.allColumns.get(i);
+ if (i > 0)
+ b.append(", ");
+ b.append(spec.name);
+ }
if (includeWriteTime)
{
for (ColumnSpec<?> column : schema.staticColumns)
- select.writeTime(column.name);
+ b.append(", ")
+ .append("writetime(")
+ .append(column.name)
+ .append(")");
for (ColumnSpec<?> column : schema.regularColumns)
- select.writeTime(column.name);
+ b.append(", ")
+ .append("writetime(")
+ .append(column.name)
+ .append(")");
}
- Select.Where where = select.from(schema.keyspace, schema.table).where();
- List<Object> bindings = new ArrayList<>();
+ b.append(" FROM ")
+ .append(schema.keyspace)
+ .append(".")
+ .append(schema.table)
+ .append(" WHERE ");
- addRelations(schema, where, bindings, pd, relations);
- addOrderBy(schema, where, reverse);
+ List<Object> bindings = new ArrayList<>();
+ schema.inflateRelations(pd,
+ relations,
+ new SchemaSpec.AddRelationCallback()
+ {
+ boolean isFirst = true;
+ public void accept(ColumnSpec<?> spec, Relation.RelationKind kind, Object value)
+ {
+ if (isFirst)
+ isFirst = false;
+ else
+ b.append(" AND ");
+ b.append(kind.getClause(spec));
+ bindings.add(value);
+ }
+ });
+ addOrderBy(schema, b, reverse);
+ b.append(";");
Object[] bindingsArr = bindings.toArray(new Object[bindings.size()]);
- return new CompiledStatement(where.toString(), bindingsArr);
+ return new CompiledStatement(b.toString(), bindingsArr);
}
public static CompiledStatement count(SchemaSpec schema, long pd)
{
- Select.Selection select = QueryBuilder.select();
- select.countAll();
-
- Select.Where where = select.from(schema.keyspace, schema.table).where();
- List<Object> bindings = new ArrayList<>(schema.partitionKeys.size());
+ StringBuilder b = new StringBuilder();
+ b.append("SELECT count(*) ");
- addRelations(schema, where, bindings, pd, Collections.emptyList());
+ b.append(" FROM ")
+ .append(schema.keyspace)
+ .append(".")
+ .append(schema.table)
+ .append(" WHERE ");
- Object[] bindingsArr = bindings.toArray(new Object[bindings.size()]);
- return new CompiledStatement(where.toString(), bindingsArr);
- }
+ List<Object> bindings = new ArrayList<>(schema.partitionKeys.size());
- private static void addRelations(SchemaSpec schema, Select.Where where, List<Object> bindings, long pd, List<Relation> relations)
- {
schema.inflateRelations(pd,
- relations,
- (spec, kind, value) -> {
- where.and(kind.getClause(spec));
- bindings.add(value);
+ Collections.emptyList(),
+ new SchemaSpec.AddRelationCallback()
+ {
+ boolean isFirst = true;
+ public void accept(ColumnSpec<?> spec, Relation.RelationKind kind, Object value)
+ {
+ if (isFirst)
+ isFirst = false;
+ else
+ b.append(" AND ");
+ b.append(kind.getClause(spec));
+ bindings.add(value);
+ }
});
+
+ Object[] bindingsArr = bindings.toArray(new Object[bindings.size()]);
+ return new CompiledStatement(b.toString(), bindingsArr);
}
- private static void addOrderBy(SchemaSpec schema, Select.Where whereClause, boolean reverse)
+ private static void addOrderBy(SchemaSpec schema, StringBuilder b, boolean reverse)
{
if (reverse && schema.clusteringKeys.size() > 0)
{
- Ordering[] ordering = new Ordering[schema.clusteringKeys.size()];
+ b.append(" ORDER BY ");
for (int i = 0; i < schema.clusteringKeys.size(); i++)
{
ColumnSpec<?> c = schema.clusteringKeys.get(i);
- ordering[i] = c.isReversed() ? QueryBuilder.asc(c.name) : QueryBuilder.desc(c.name);
+ if (i > 0)
+ b.append(", ");
+ b.append(c.isReversed() ? asc(c.name) : desc(c.name));
}
- whereClause.orderBy(ordering);
}
}
+ public static String asc(String name)
+ {
+ return name + " ASC";
+ }
+
+ public static String desc(String name)
+ {
+ return name + " DESC";
+ }
+
public static ResultSetRow resultSetToRow(SchemaSpec schema, OpSelectors.MonotonicClock clock, Object[] result)
{
Object[] partitionKey = new Object[schema.partitionKeys.size()];
diff --git a/harry-core/src/harry/model/clock/ApproximateMonotonicClock.java b/harry-core/src/harry/model/clock/ApproximateMonotonicClock.java
index 59dd47e..1a64500 100644
--- a/harry-core/src/harry/model/clock/ApproximateMonotonicClock.java
+++ b/harry-core/src/harry/model/clock/ApproximateMonotonicClock.java
@@ -25,9 +25,8 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicLongArray;
import java.util.concurrent.locks.LockSupport;
-import com.google.common.annotations.VisibleForTesting;
-
import harry.core.Configuration;
+import harry.core.VisibleForTesting;
import harry.model.OpSelectors;
/**
diff --git a/harry-core/src/harry/model/clock/OffsetClock.java b/harry-core/src/harry/model/clock/OffsetClock.java
index 9f40a64..8c25394 100644
--- a/harry-core/src/harry/model/clock/OffsetClock.java
+++ b/harry-core/src/harry/model/clock/OffsetClock.java
@@ -20,6 +20,9 @@ package harry.model.clock;
import java.util.concurrent.atomic.AtomicLong;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
import harry.core.Configuration;
import harry.model.OpSelectors;
@@ -58,4 +61,21 @@ public class OffsetClock implements OpSelectors.MonotonicClock
{
throw new RuntimeException("not implemented");
}
+
+ @JsonTypeName("offset")
+ public static class OffsetClockConfiguration implements Configuration.ClockConfiguration
+ {
+ public final long offset;
+
+ @JsonCreator
+ public OffsetClockConfiguration(@JsonProperty("offset") int offset)
+ {
+ this.offset = offset;
+ }
+
+ public OpSelectors.MonotonicClock make()
+ {
+ return new OffsetClock(offset);
+ }
+ }
}
diff --git a/harry-core/src/harry/model/sut/PrintlnSut.java b/harry-core/src/harry/model/sut/PrintlnSut.java
index ad14b0b..f1b310e 100644
--- a/harry-core/src/harry/model/sut/PrintlnSut.java
+++ b/harry-core/src/harry/model/sut/PrintlnSut.java
@@ -21,6 +21,10 @@ package harry.model.sut;
import java.util.Arrays;
import java.util.concurrent.CompletableFuture;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import harry.core.Configuration;
+
public class PrintlnSut implements SystemUnderTest
{
public boolean isShutdown()
@@ -46,4 +50,18 @@ public class PrintlnSut implements SystemUnderTest
return CompletableFuture.supplyAsync(() -> execute(statement, cl, bindings),
Runnable::run);
}
+
+ @JsonTypeName("println")
+ public static class PrintlnSutConfiguration implements Configuration.SutConfiguration
+ {
+ @JsonCreator
+ public PrintlnSutConfiguration()
+ {
+
+ }
+ public SystemUnderTest make()
+ {
+ return new PrintlnSut();
+ }
+ }
}
diff --git a/harry-core/src/harry/operations/DeleteHelper.java b/harry-core/src/harry/operations/DeleteHelper.java
index 6f04bad..f1b6983 100644
--- a/harry-core/src/harry/operations/DeleteHelper.java
+++ b/harry-core/src/harry/operations/DeleteHelper.java
@@ -23,11 +23,8 @@ import java.util.Collections;
import java.util.List;
import java.util.function.IntConsumer;
-import com.datastax.driver.core.querybuilder.Delete;
-import com.datastax.driver.core.querybuilder.QueryBuilder;
import harry.ddl.ColumnSpec;
import harry.ddl.SchemaSpec;
-import harry.runner.LoggingPartitionVisitor;
import harry.util.BitSet;
public class DeleteHelper
@@ -130,40 +127,50 @@ public class DeleteHelper
BitSet mask,
long ts)
{
- Delete delete;
- if (columnsToDelete == null)
- delete = QueryBuilder.delete().from(schema.keyspace, schema.table);
- else
+ StringBuilder b = new StringBuilder();
+ b.append("DELETE ");
+ if (columnsToDelete != null)
{
assert mask != null;
assert relations == null || relations.stream().allMatch((r) -> r.kind == Relation.RelationKind.EQ);
- delete = QueryBuilder.delete(columnNames(schema.allColumns, columnsToDelete, mask))
- .from(schema.keyspace, schema.table);
+ String[] names = columnNames(schema.allColumns, columnsToDelete, mask);
+ for (int i = 0; i < names.length; i++)
+ {
+ if (i > 0)
+ b.append(", ");
+ b.append(names[i]);
+ }
+ b.append(" ");
}
+ b.append("FROM ")
+ .append(schema.keyspace).append(".").append(schema.table)
+ .append(" USING TIMESTAMP ")
+ .append(ts)
+ .append(" WHERE ");
- Delete.Where where = delete.where();
List<Object> bindings = new ArrayList<>();
- addRelations(schema, where, bindings, pd, relations);
- delete.using(QueryBuilder.timestamp(ts));
- delete.setForceNoValues(true);
- Object[] bindingsArr = bindings.toArray(new Object[bindings.size()]);
- String compiled = delete.getQueryString();
- if (compiled.contains("built query (could not generate with default codec registry:"))
- throw new IllegalArgumentException(String.format("Could not generate the query: %s. Bindings: (%s)",
- delete,
- CompiledStatement.bindingsToString(bindingsArr)));
- return new CompiledStatement(compiled, bindingsArr);
- }
-
- private static void addRelations(SchemaSpec schema, Delete.Where where, List<Object> bindings, long pd, List<Relation> relations)
- {
schema.inflateRelations(pd,
relations,
- (spec, kind, value) -> {
- where.and(kind.getClause(spec));
- bindings.add(value);
+ new SchemaSpec.AddRelationCallback()
+ {
+ boolean isFirst = true;
+ public void accept(ColumnSpec<?> spec, Relation.RelationKind kind, Object value)
+ {
+ if (isFirst)
+ isFirst = false;
+ else
+ b.append(" AND ");
+ b.append(kind.getClause(spec));
+ bindings.add(value);
+ }
});
+
+ b.append(";");
+
+ Object[] bindingsArr = bindings.toArray(new Object[bindings.size()]);
+
+ return new CompiledStatement(b.toString(), bindingsArr);
}
private static String[] columnNames(List<ColumnSpec<?>> columns, BitSet selectedColumns, BitSet mask)
diff --git a/harry-core/src/harry/operations/Relation.java b/harry-core/src/harry/operations/Relation.java
index 19db0b4..87487d9 100644
--- a/harry-core/src/harry/operations/Relation.java
+++ b/harry-core/src/harry/operations/Relation.java
@@ -21,16 +21,8 @@ package harry.operations;
import java.util.ArrayList;
import java.util.List;
-import com.datastax.driver.core.querybuilder.Clause;
import harry.ddl.ColumnSpec;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.bindMarker;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.eq;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.gt;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.gte;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.lt;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.lte;
-
public class Relation
{
public final RelationKind kind;
@@ -62,9 +54,9 @@ public class Relation
return columnSpec.name;
}
- public Clause toClause()
+ public String toClause()
{
- return kind.getClause(column(), bindMarker());
+ return kind.getClause(column());
}
public String toString()
@@ -101,7 +93,7 @@ public class Relation
public static void addRelation(long[] key, List<ColumnSpec<?>> columnSpecs, List<Relation> relations, RelationKind kind)
{
assert key.length == columnSpecs.size() :
- String.format("Key size (%d) should equal to column spec size (%d)", key.length, columnSpecs.size());
+ String.format("Key size (%d) should equal to column spec size (%d). Specs: %s", key.length, columnSpecs.size(), columnSpecs);
for (int i = 0; i < key.length; i++)
{
ColumnSpec<?> spec = columnSpecs.get(i);
@@ -113,17 +105,6 @@ public class Relation
{
LT
{
- @Override
- public Clause getClause(String name, Object obj)
- {
- return lt(name, obj);
- }
-
- public Clause getClause(List<String> name, List<Object> obj)
- {
- return lt(name, obj);
- }
-
public boolean isNegatable()
{
return true;
@@ -156,17 +137,6 @@ public class Relation
},
GT
{
- @Override
- public Clause getClause(String name, Object obj)
- {
- return gt(name, obj);
- }
-
- public Clause getClause(List<String> name, List<Object> obj)
- {
- return gt(name, obj);
- }
-
public boolean isNegatable()
{
return true;
@@ -199,17 +169,6 @@ public class Relation
},
LTE
{
- @Override
- public Clause getClause(String name, Object obj)
- {
- return lte(name, obj);
- }
-
- public Clause getClause(List<String> name, List<Object> obj)
- {
- return lt(name, obj);
- }
-
public boolean isNegatable()
{
return true;
@@ -242,17 +201,6 @@ public class Relation
},
GTE
{
- @Override
- public Clause getClause(String name, Object obj)
- {
- return gte(name, obj);
- }
-
- public Clause getClause(List<String> name, List<Object> obj)
- {
- return gte(name, obj);
- }
-
public boolean isNegatable()
{
return true;
@@ -285,17 +233,6 @@ public class Relation
},
EQ
{
- @Override
- public Clause getClause(String name, Object obj)
- {
- return eq(name, obj);
- }
-
- public Clause getClause(List<String> name, List<Object> obj)
- {
- return eq(name, obj);
- }
-
public boolean isNegatable()
{
return false;
@@ -329,14 +266,15 @@ public class Relation
public abstract boolean match(LongComparator comparator, long l, long r);
- public abstract Clause getClause(String name, Object obj);
-
- public Clause getClause(ColumnSpec<?> spec)
+ public String getClause(String name)
{
- return getClause(spec.name, bindMarker());
+ return String.format("%s %s ?", name, toString());
}
- public abstract Clause getClause(List<String> name, List<Object> obj);
+ public String getClause(ColumnSpec<?> spec)
+ {
+ return getClause(spec.name);
+ }
public abstract boolean isNegatable();
diff --git a/harry-core/src/harry/operations/WriteHelper.java b/harry-core/src/harry/operations/WriteHelper.java
index a0b7565..084f931 100644
--- a/harry-core/src/harry/operations/WriteHelper.java
+++ b/harry-core/src/harry/operations/WriteHelper.java
@@ -18,21 +18,13 @@
package harry.operations;
+import java.util.Arrays;
import java.util.List;
import harry.ddl.ColumnSpec;
import harry.ddl.SchemaSpec;
import harry.generators.DataGenerators;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.bindMarker;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.eq;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.in;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.insertInto;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.set;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.timestamp;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.truncate;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.update;
-
public class WriteHelper
{
public static CompiledStatement inflateInsert(SchemaSpec schema,
@@ -48,117 +40,127 @@ public class WriteHelper
Object[] regularColumns = schema.inflateRegularColumns(vds);
Object[] bindings = new Object[schema.allColumns.size()];
- int bindingsCount = 0;
- com.datastax.driver.core.querybuilder.Insert insert = insertInto(schema.keyspace,
- schema.table);
- bindingsCount += addValue(insert, bindings, schema.partitionKeys, partitionKey, bindingsCount);
- bindingsCount += addValue(insert, bindings, schema.clusteringKeys, clusteringKey, bindingsCount);
+ StringBuilder b = new StringBuilder();
+ b.append("INSERT INTO ")
+ .append(schema.keyspace)
+ .append('.')
+ .append(schema.table)
+ .append(" (");
+
+ int bindingsCount = 0;
+ bindingsCount += appendStatements(b, bindings, schema.partitionKeys, partitionKey, bindingsCount, true, ",", "%s");
+ bindingsCount += appendStatements(b, bindings, schema.clusteringKeys, clusteringKey, bindingsCount, false, ",", "%s");
+ bindingsCount += appendStatements(b, bindings, schema.regularColumns, regularColumns, bindingsCount, false, ",", "%s");
if (staticColumns != null)
- bindingsCount += addValue(insert, bindings, schema.staticColumns, staticColumns, bindingsCount);
- bindingsCount += addValue(insert, bindings, schema.regularColumns, regularColumns, bindingsCount);
+ bindingsCount += appendStatements(b, bindings, schema.staticColumns, staticColumns, bindingsCount, false, ",", "%s");
- insert.using(timestamp(timestamp));
+ b.append(") VALUES (");
- // Some of the values were unset
- if (bindingsCount != bindings.length)
+ for (int i = 0; i < bindingsCount; i++)
{
- Object[] tmp = new Object[bindingsCount];
- System.arraycopy(bindings, 0, tmp, 0, bindingsCount);
- bindings = tmp;
+ if (i > 0)
+ b.append(", ");
+ b.append("?");
}
- return CompiledStatement.create(insert.toString(), bindings);
+ b.append(") USING TIMESTAMP ")
+ .append(timestamp);
+
+ return new CompiledStatement(b.toString(), adjustArraySize(bindings, bindingsCount));
}
- public static boolean allUnset(long[] descriptors)
+ public static Object[] adjustArraySize(Object[] bindings, int bindingsCount)
{
- for (long descriptor : descriptors)
+ if (bindingsCount != bindings.length)
{
- if (descriptor != DataGenerators.UNSET_DESCR)
- return false;
+ Object[] tmp = new Object[bindingsCount];
+ System.arraycopy(bindings, 0, tmp, 0, bindingsCount);
+ bindings = tmp;
}
- return true;
- }
- private static int addValue(com.datastax.driver.core.querybuilder.Insert insert,
- Object[] bindings,
- List<ColumnSpec<?>> columns,
- Object[] data,
- int bound)
- {
- assert data.length == columns.size();
-
- int bindingsCount = 0;
- for (int i = 0; i < data.length; i++)
- {
- if (data[i] == DataGenerators.UNSET_VALUE)
- continue;
-
- insert.value(columns.get(i).name, bindMarker());
- bindings[bound + bindingsCount] = data[i];
- bindingsCount++;
- }
-
- return bindingsCount;
+ return bindings;
}
public static CompiledStatement inflateUpdate(SchemaSpec schema,
long pd,
long cd,
long[] vds,
+ long[] sds,
long timestamp)
{
Object[] partitionKey = schema.inflatePartitionKey(pd);
Object[] clusteringKey = schema.inflateClusteringKey(cd);
+ Object[] staticColumns = sds == null ? null : schema.inflateStaticColumns(sds);
Object[] regularColumns = schema.inflateRegularColumns(vds);
Object[] bindings = new Object[schema.allColumns.size()];
- int bindingsCount = 0;
- com.datastax.driver.core.querybuilder.Update update = update(schema.keyspace,
- schema.table);
- bindingsCount += addWith(update, bindings, schema.regularColumns, regularColumns, bindingsCount);
- bindingsCount += addWhere(update, bindings, schema.partitionKeys, partitionKey, bindingsCount);
- bindingsCount += addWhere(update, bindings, schema.clusteringKeys, clusteringKey, bindingsCount);
+ StringBuilder b = new StringBuilder();
+ b.append("UPDATE ")
+ .append(schema.keyspace)
+ .append('.')
+ .append(schema.table)
+ .append(" USING TIMESTAMP ")
+ .append(timestamp)
+ .append(" SET ");
- update.using(timestamp(timestamp));
- // TODO: TTL
- // ttl.ifPresent(ts -> update.using(ttl(ts)));
+ int bindingsCount = 0;
+ bindingsCount += addSetStatements(b, bindings, schema.regularColumns, regularColumns, bindingsCount);
+ if (staticColumns != null)
+ bindingsCount += addSetStatements(b, bindings, schema.staticColumns, staticColumns, bindingsCount);
+
+ assert bindingsCount > 0 : "Can not have an UPDATE statement without any updates";
+ b.append(" WHERE ");
- return CompiledStatement.create(update.toString(), bindings);
+ bindingsCount += addWhereStatements(b, bindings, schema.partitionKeys, partitionKey, bindingsCount, true);
+ bindingsCount += addWhereStatements(b, bindings, schema.clusteringKeys, clusteringKey, bindingsCount, false);
+ b.append(";");
+ return new CompiledStatement(b.toString(), adjustArraySize(bindings, bindingsCount));
}
- private static int addWith(com.datastax.driver.core.querybuilder.Update update,
- Object[] bindings,
- List<ColumnSpec<?>> columns,
- Object[] data,
- int bound)
+ private static int addSetStatements(StringBuilder b,
+ Object[] bindings,
+ List<ColumnSpec<?>> columns,
+ Object[] values,
+ int bound)
{
- assert data.length == columns.size();
-
- for (int i = 0; i < data.length; i++)
- {
- update.with(set(columns.get(i).name, bindMarker()));
- bindings[bound + i] = data[i];
- }
-
- return data.length;
+ return appendStatements(b, bindings, columns, values, bound, bound == 0, ", ", "%s = ?");
}
- private static int addWhere(com.datastax.driver.core.querybuilder.Update update,
- Object[] bindings,
- List<ColumnSpec<?>> columns,
- Object[] data,
- int bound)
+ private static int addWhereStatements(StringBuilder b,
+ Object[] bindings,
+ List<ColumnSpec<?>> columns,
+ Object[] values,
+ int bound,
+ boolean firstStatement)
{
- assert data.length == columns.size();
+ return appendStatements(b, bindings, columns, values, bound, firstStatement, " AND ", "%s = ?");
+ }
- for (int i = 0; i < data.length; i++)
+ private static int appendStatements(StringBuilder b,
+ Object[] allBindings,
+ List<ColumnSpec<?>> columns,
+ Object[] values,
+ int bound,
+ boolean firstStatement,
+ String separator,
+ String nameFormatter)
+ {
+ int bindingsCount = 0;
+ for (int i = 0; i < values.length; i++)
{
- update.where().and(eq(columns.get(i).name, bindMarker()));
- bindings[bound + i] = data[i];
- }
+ Object value = values[i];
+ if (value == DataGenerators.UNSET_VALUE)
+ continue;
+
+ ColumnSpec<?> column = columns.get(i);
+ if (bindingsCount > 0 || !firstStatement)
+ b.append(separator);
- return data.length;
+ b.append(String.format(nameFormatter, column.name));
+ allBindings[bound + bindingsCount] = value;
+ bindingsCount++;
+ }
+ return bindingsCount;
}
}
\ No newline at end of file
diff --git a/harry-core/src/harry/reconciler/Reconciler.java b/harry-core/src/harry/reconciler/Reconciler.java
index ca4772c..a9709b3 100644
--- a/harry-core/src/harry/reconciler/Reconciler.java
+++ b/harry-core/src/harry/reconciler/Reconciler.java
@@ -125,8 +125,10 @@ public class Reconciler
hadPartitionDeletion = true;
break;
- case WRITE_WITH_STATICS:
- case WRITE:
+ case INSERT_WITH_STATICS:
+ case INSERT:
+ case UPDATE:
+ case UPDATE_WITH_STATICS:
if (debugCd != -1 && cd == debugCd)
logger.info("Writing {} ({}) at {}/{}", cd, opType, lts, opId);
writes.add(opId);
@@ -164,12 +166,14 @@ public class Reconciler
switch (opType)
{
- case WRITE_WITH_STATICS:
+ case INSERT_WITH_STATICS:
+ case UPDATE_WITH_STATICS:
// We could apply static columns during the first iteration, but it's more convenient
// to reconcile static-level deletions.
partitionState.writeStaticRow(descriptorSelector.sds(pd, cd, lts, opId, schema),
lts);
- case WRITE:
+ case INSERT:
+ case UPDATE:
if (!query.match(cd))
{
if (debugCd != -1 && cd == debugCd)
@@ -189,7 +193,8 @@ public class Reconciler
partitionState.write(cd,
descriptorSelector.vds(pd, cd, lts, opId, schema),
- lts);
+ lts,
+ opType == OpSelectors.OperationKind.INSERT || opType == OpSelectors.OperationKind.INSERT_WITH_STATICS);
break;
default:
throw new IllegalStateException();
@@ -206,7 +211,8 @@ public class Reconciler
switch (opType)
{
case DELETE_COLUMN_WITH_STATICS:
- partitionState.deleteStaticColumns(schema.staticColumnsOffset,
+ partitionState.deleteStaticColumns(lts,
+ schema.staticColumnsOffset,
descriptorSelector.columnMask(pd, lts, opId),
schema.staticColumnsMask());
case DELETE_COLUMN:
@@ -227,7 +233,8 @@ public class Reconciler
}
}
- partitionState.deleteRegularColumns(cd,
+ partitionState.deleteRegularColumns(lts,
+ cd,
schema.regularColumnsOffset,
descriptorSelector.columnMask(pd, lts, opId),
schema.regularColumnsMask());
@@ -270,14 +277,15 @@ public class Reconciler
long lts)
{
if (staticRow != null)
- staticRow = updateRowState(staticRow, schema.staticColumns, STATIC_CLUSTERING, staticVds, lts);
+ staticRow = updateRowState(staticRow, schema.staticColumns, STATIC_CLUSTERING, staticVds, lts, false);
}
private void write(long cd,
long[] vds,
- long lts)
+ long lts,
+ boolean writeParimaryKeyLiveness)
{
- rows.compute(cd, (cd_, current) -> updateRowState(current, schema.regularColumns, cd, vds, lts));
+ rows.compute(cd, (cd_, current) -> updateRowState(current, schema.regularColumns, cd, vds, lts, writeParimaryKeyLiveness));
}
private void delete(Ranges.Range range,
@@ -304,14 +312,19 @@ public class Reconciler
private void delete(long cd,
long lts)
{
- rows.remove(cd);
+ RowState state = rows.remove(cd);
+ if (state != null)
+ {
+ for (long v : state.lts)
+ assert lts >= v : String.format("Attempted to remove a row with a tombstone that has older timestamp (%d): %s", lts, state);
+ }
}
public boolean isEmpty()
{
return rows.isEmpty();
}
- private RowState updateRowState(RowState currentState, List<ColumnSpec<?>> columns, long cd, long[] vds, long lts)
+ private RowState updateRowState(RowState currentState, List<ColumnSpec<?>> columns, long cd, long[] vds, long lts, boolean writePrimaryKeyLiveness)
{
if (currentState == null)
{
@@ -359,24 +372,30 @@ public class Reconciler
}
}
+ if (writePrimaryKeyLiveness)
+ currentState.hasPrimaryKeyLivenessInfo = true;
+
return currentState;
}
- private void deleteRegularColumns(long cd, int columnOffset, BitSet columns, BitSet mask)
+ private void deleteRegularColumns(long lts, long cd, int columnOffset, BitSet columns, BitSet mask)
{
- deleteColumns(rows.get(cd), columnOffset, columns, mask);
+ deleteColumns(lts, rows.get(cd), columnOffset, columns, mask);
}
- private void deleteStaticColumns(int columnOffset, BitSet columns, BitSet mask)
+ private void deleteStaticColumns(long lts, int columnOffset, BitSet columns, BitSet mask)
{
- deleteColumns(staticRow, columnOffset, columns, mask);
+ deleteColumns(lts, staticRow, columnOffset, columns, mask);
}
- private void deleteColumns(RowState state, int columnOffset, BitSet columns, BitSet mask)
+ private void deleteColumns(long lts, RowState state, int columnOffset, BitSet columns, BitSet mask)
{
if (state == null)
return;
+ //TODO: optimise by iterating over the columns that were removed by this deletion
+ //TODO: optimise final decision to fully remove the column by counting a number of set/unset columns
+ boolean allNil = true;
for (int i = 0; i < state.vds.length; i++)
{
if (columns.isSet(columnOffset + i, mask))
@@ -384,7 +403,14 @@ public class Reconciler
state.vds[i] = NIL_DESCR;
state.lts[i] = NO_TIMESTAMP;
}
+ else if (state.vds[i] != NIL_DESCR)
+ {
+ allNil = false;
+ }
}
+
+ if (state.cd != STATIC_CLUSTERING && allNil & !state.hasPrimaryKeyLivenessInfo)
+ delete(state.cd, lts);
}
private void deletePartition(long lts)
@@ -449,6 +475,7 @@ public class Reconciler
public static class RowState
{
+ public boolean hasPrimaryKeyLivenessInfo = false;
public final long cd;
public final long[] vds;
public final long[] lts;
diff --git a/harry-core/src/harry/runner/CorruptingPartitionVisitor.java b/harry-core/src/harry/runner/CorruptingPartitionVisitor.java
index ca5bb93..d079cf3 100644
--- a/harry-core/src/harry/runner/CorruptingPartitionVisitor.java
+++ b/harry-core/src/harry/runner/CorruptingPartitionVisitor.java
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package harry.runner;
import java.util.Random;
diff --git a/harry-core/src/harry/runner/DataTracker.java b/harry-core/src/harry/runner/DataTracker.java
index d5825fb..dab6fc9 100644
--- a/harry-core/src/harry/runner/DataTracker.java
+++ b/harry-core/src/harry/runner/DataTracker.java
@@ -18,6 +18,8 @@
package harry.runner;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonTypeName;
import harry.core.Configuration;
public interface DataTracker
@@ -35,6 +37,7 @@ public interface DataTracker
}
public static DataTracker NO_OP = new NoOpDataTracker();
+
class NoOpDataTracker implements DataTracker
{
private NoOpDataTracker() {}
diff --git a/harry-core/src/harry/runner/DefaultDataTracker.java b/harry-core/src/harry/runner/DefaultDataTracker.java
index 1b55482..7b1412a 100644
--- a/harry-core/src/harry/runner/DefaultDataTracker.java
+++ b/harry-core/src/harry/runner/DefaultDataTracker.java
@@ -23,11 +23,11 @@ import java.util.List;
import java.util.concurrent.PriorityBlockingQueue;
import java.util.concurrent.atomic.AtomicLong;
-import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import harry.core.Configuration;
+import harry.core.VisibleForTesting;
public class DefaultDataTracker implements DataTracker
{
diff --git a/harry-core/src/harry/runner/MutatingPartitionVisitor.java b/harry-core/src/harry/runner/MutatingPartitionVisitor.java
index 02aa6a1..4df793e 100644
--- a/harry-core/src/harry/runner/MutatingPartitionVisitor.java
+++ b/harry-core/src/harry/runner/MutatingPartitionVisitor.java
@@ -127,6 +127,7 @@ public class MutatingPartitionVisitor extends AbstractPartitionVisitor
if (sut.isShutdown())
throw new IllegalStateException("System under test is shut down");
+ // TODO: limit a number of retries
sut.executeAsync(statement.cql(), SystemUnderTest.ConsistencyLevel.QUORUM, statement.bindings())
.whenComplete((res, t) -> {
if (t != null)
diff --git a/harry-core/src/harry/runner/MutatingRowVisitor.java b/harry-core/src/harry/runner/MutatingRowVisitor.java
index b928df7..a14fc96 100644
--- a/harry-core/src/harry/runner/MutatingRowVisitor.java
+++ b/harry-core/src/harry/runner/MutatingRowVisitor.java
@@ -20,6 +20,7 @@ package harry.runner;
import harry.core.MetricReporter;
import harry.core.Run;
+import harry.core.VisibleForTesting;
import harry.ddl.SchemaSpec;
import harry.model.OpSelectors;
import harry.operations.CompiledStatement;
@@ -37,21 +38,35 @@ public class MutatingRowVisitor implements Operation
public MutatingRowVisitor(Run run)
{
- this.metricReporter = run.metricReporter;
- this.schema = run.schemaSpec;
- this.clock = run.clock;
- this.descriptorSelector = run.descriptorSelector;
- this.rangeSelector = run.rangeSelector;
+ this(run.schemaSpec,
+ run.clock,
+ run.descriptorSelector,
+ run.rangeSelector,
+ run.metricReporter);
}
- public CompiledStatement write(long lts, long pd, long cd, long opId)
+ @VisibleForTesting
+ public MutatingRowVisitor(SchemaSpec schema,
+ OpSelectors.MonotonicClock clock,
+ OpSelectors.DescriptorSelector descriptorSelector,
+ QueryGenerator rangeSelector,
+ MetricReporter metricReporter)
+ {
+ this.metricReporter = metricReporter;
+ this.schema = schema;
+ this.clock = clock;
+ this.descriptorSelector = descriptorSelector;
+ this.rangeSelector = rangeSelector;
+ }
+
+ public CompiledStatement insert(long lts, long pd, long cd, long opId)
{
metricReporter.insert();
long[] vds = descriptorSelector.vds(pd, cd, lts, opId, schema);
return WriteHelper.inflateInsert(schema, pd, cd, vds, null, clock.rts(lts));
}
- public CompiledStatement writeWithStatics(long lts, long pd, long cd, long opId)
+ public CompiledStatement insertWithStatics(long lts, long pd, long cd, long opId)
{
metricReporter.insert();
long[] vds = descriptorSelector.vds(pd, cd, lts, opId, schema);
@@ -59,6 +74,21 @@ public class MutatingRowVisitor implements Operation
return WriteHelper.inflateInsert(schema, pd, cd, vds, sds, clock.rts(lts));
}
+ public CompiledStatement update(long lts, long pd, long cd, long opId)
+ {
+ metricReporter.insert();
+ long[] vds = descriptorSelector.vds(pd, cd, lts, opId, schema);
+ return WriteHelper.inflateUpdate(schema, pd, cd, vds, null, clock.rts(lts));
+ }
+
+ public CompiledStatement updateWithStatics(long lts, long pd, long cd, long opId)
+ {
+ metricReporter.insert();
+ long[] vds = descriptorSelector.vds(pd, cd, lts, opId, schema);
+ long[] sds = descriptorSelector.sds(pd, cd, lts, opId, schema);
+ return WriteHelper.inflateUpdate(schema, pd, cd, vds, sds, clock.rts(lts));
+ }
+
public CompiledStatement deleteColumn(long lts, long pd, long cd, long opId)
{
metricReporter.columnDelete();
diff --git a/harry-core/src/harry/runner/Operation.java b/harry-core/src/harry/runner/Operation.java
index e56be4c..f44f3bd 100644
--- a/harry-core/src/harry/runner/Operation.java
+++ b/harry-core/src/harry/runner/Operation.java
@@ -35,12 +35,16 @@ public interface Operation
{
// TODO: switch to EnumMap
// TODO: pluggable capabilities; OperationKind can/should bear its own logic
- case WRITE:
- return write(lts, pd, cd, opId);
+ case INSERT:
+ return insert(lts, pd, cd, opId);
+ case UPDATE:
+ return update(lts, pd, cd, opId);
case DELETE_ROW:
return deleteRow(lts, pd, cd, opId);
- case WRITE_WITH_STATICS:
- return writeWithStatics(lts, pd, cd, opId);
+ case INSERT_WITH_STATICS:
+ return insertWithStatics(lts, pd, cd, opId);
+ case UPDATE_WITH_STATICS:
+ return updateWithStatics(lts, pd, cd, opId);
case DELETE_PARTITION:
return deletePartition(lts, pd, opId);
case DELETE_COLUMN:
@@ -56,7 +60,11 @@ public interface Operation
}
}
- CompiledStatement write(long lts, long pd, long cd, long opId);
+ CompiledStatement insert(long lts, long pd, long cd, long opId);
+ CompiledStatement update(long lts, long pd, long cd, long opId);
+
+ CompiledStatement insertWithStatics(long lts, long pd, long cd, long opId);
+ CompiledStatement updateWithStatics(long lts, long pd, long cd, long opId);
CompiledStatement deleteColumn(long lts, long pd, long cd, long opId);
@@ -66,8 +74,6 @@ public interface Operation
CompiledStatement deletePartition(long lts, long pd, long opId);
- CompiledStatement writeWithStatics(long lts, long pd, long cd, long opId);
-
CompiledStatement deleteRange(long lts, long pd, long opId);
CompiledStatement deleteSlice(long lts, long pd, long opId);
diff --git a/harry-core/src/harry/runner/QueryGenerator.java b/harry-core/src/harry/runner/QueryGenerator.java
index 65b21ac..829cf8b 100644
--- a/harry-core/src/harry/runner/QueryGenerator.java
+++ b/harry-core/src/harry/runner/QueryGenerator.java
@@ -317,7 +317,7 @@ public class QueryGenerator
// TODO: one of the ways to get rid of garbage here, and potentially even simplify the code is to
// simply return bounds here. After bounds are created, we slice them and generate query right
// from the bounds. In this case, we can even say that things like -inf/+inf are special values,
- // and use them as placeholdrs. Also, it'll be easier to manipulate relations.
+ // and use them as placeholders. Also, it'll be easier to manipulate relations.
return new Query.ClusteringRangeQuery(Query.QueryKind.CLUSTERING_RANGE,
pd,
stitchedMin,
diff --git a/harry-core/src/harry/util/BitSet.java b/harry-core/src/harry/util/BitSet.java
index 3c70052..4dc8823 100644
--- a/harry-core/src/harry/util/BitSet.java
+++ b/harry-core/src/harry/util/BitSet.java
@@ -178,7 +178,7 @@ public interface BitSet
public boolean isSet(int idx)
{
- assert idx < size();
+ assert idx < size() : String.format("Trying to query the bit (%s) outside the range of bitset (%s)", idx, size());
return BitSet.isSet(bits, idx);
}
diff --git a/harry-core/src/harry/util/TestRunner.java b/harry-core/src/harry/util/TestRunner.java
index 9abea62..4349fd2 100644
--- a/harry-core/src/harry/util/TestRunner.java
+++ b/harry-core/src/harry/util/TestRunner.java
@@ -44,19 +44,24 @@ public class TestRunner
public static <T1, T2> void test(Generator<T1> gen1,
Function<T1, Generator<T2>> gen2,
- Consumer<T2> validate)
+ ThrowingConsumer<T2> validate) throws Throwable
{
test(gen1,
(v1) -> test(gen2.apply(v1), validate));
}
public static <T1> void test(Generator<T1> gen1,
- Consumer<T1> validate)
+ ThrowingConsumer<T1> validate) throws Throwable
{
for (int i = 0; i < CYCLES; i++)
{
validate.accept(gen1.generate(rand));
}
}
+
+ public static interface ThrowingConsumer<T>
+ {
+ void accept(T t) throws Throwable;
+ }
}
diff --git a/harry-core/test/harry/model/OpSelectorsTest.java b/harry-core/test/harry/model/OpSelectorsTest.java
index b291cbc..266aeb9 100644
--- a/harry-core/test/harry/model/OpSelectorsTest.java
+++ b/harry-core/test/harry/model/OpSelectorsTest.java
@@ -25,7 +25,6 @@ import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
-import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
import java.util.function.BiConsumer;
@@ -39,7 +38,6 @@ import harry.core.Run;
import harry.ddl.ColumnSpec;
import harry.ddl.SchemaGenerators;
import harry.ddl.SchemaSpec;
-import harry.generators.RngUtils;
import harry.generators.Surjections;
import harry.generators.distribution.Distribution;
import harry.model.clock.OffsetClock;
@@ -186,10 +184,11 @@ public class OpSelectorsTest
OpSelectors.PdSelector pdSelector = new OpSelectors.DefaultPdSelector(rng, 10, 10);
OpSelectors.DescriptorSelector ckSelector = new OpSelectors.DefaultDescriptorSelector(rng,
new OpSelectors.ColumnSelectorBuilder().forAll(schema, Surjections.pick(BitSet.allUnset(0))).build(),
- OpSelectors.OperationSelector.weighted(Surjections.weights(10, 10, 80),
+ OpSelectors.OperationSelector.weighted(Surjections.weights(10, 10, 40, 40),
OpSelectors.OperationKind.DELETE_ROW,
OpSelectors.OperationKind.DELETE_COLUMN,
- OpSelectors.OperationKind.WRITE),
+ OpSelectors.OperationKind.INSERT,
+ OpSelectors.OperationKind.UPDATE),
new Distribution.ConstantDistribution(2),
new Distribution.ConstantDistribution(5),
10);
@@ -217,7 +216,13 @@ public class OpSelectorsTest
PartitionVisitor partitionVisitor = new MutatingPartitionVisitor(run,
(r) -> new Operation()
{
- public CompiledStatement write(long lts, long pd, long cd, long m)
+ public CompiledStatement insert(long lts, long pd, long cd, long m)
+ {
+ consumer.accept(pd, cd);
+ return compiledStatement;
+ }
+
+ public CompiledStatement update(long lts, long pd, long cd, long opId)
{
consumer.accept(pd, cd);
return compiledStatement;
@@ -247,7 +252,13 @@ public class OpSelectorsTest
return compiledStatement;
}
- public CompiledStatement writeWithStatics(long lts, long pd, long cd, long opId)
+ public CompiledStatement insertWithStatics(long lts, long pd, long cd, long opId)
+ {
+ consumer.accept(pd, cd);
+ return compiledStatement;
+ }
+
+ public CompiledStatement updateWithStatics(long lts, long pd, long cd, long opId)
{
consumer.accept(pd, cd);
return compiledStatement;
@@ -290,10 +301,11 @@ public class OpSelectorsTest
OpSelectors.DescriptorSelector ckSelector = new OpSelectors.HierarchicalDescriptorSelector(rng,
new int[] {10, 20},
OpSelectors.columnSelectorBuilder().forAll(schema, Surjections.pick(BitSet.allUnset(0))).build(),
- OpSelectors.OperationSelector.weighted(Surjections.weights(10, 10, 80),
+ OpSelectors.OperationSelector.weighted(Surjections.weights(10, 10, 40, 40),
OpSelectors.OperationKind.DELETE_ROW,
OpSelectors.OperationKind.DELETE_COLUMN,
- OpSelectors.OperationKind.WRITE),
+ OpSelectors.OperationKind.INSERT,
+ OpSelectors.OperationKind.UPDATE),
new Distribution.ConstantDistribution(2),
new Distribution.ConstantDistribution(5),
100);
@@ -323,8 +335,10 @@ public class OpSelectorsTest
config.put(OpSelectors.OperationKind.DELETE_COLUMN, 1);
config.put(OpSelectors.OperationKind.DELETE_PARTITION, 1);
config.put(OpSelectors.OperationKind.DELETE_COLUMN_WITH_STATICS, 1);
- config.put(OpSelectors.OperationKind.WRITE_WITH_STATICS, 1000);
- config.put(OpSelectors.OperationKind.WRITE, 1000);
+ config.put(OpSelectors.OperationKind.UPDATE, 500);
+ config.put(OpSelectors.OperationKind.INSERT, 500);
+ config.put(OpSelectors.OperationKind.UPDATE_WITH_STATICS, 500);
+ config.put(OpSelectors.OperationKind.INSERT_WITH_STATICS, 500);
int[] weights = new int[config.size()];
for (int i = 0; i < config.values().size(); i++)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra-harry] 02/04: Integration improvements
Posted by if...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch CASSANDRA-16262-2
in repository https://gitbox.apache.org/repos/asf/cassandra-harry.git
commit 57053decc6b810d1148e1b7f680e54feece57d99
Author: Alex Petrov <ol...@gmail.com>
AuthorDate: Mon Jul 12 17:54:42 2021 +0200
Integration improvements
---
.../test/resources/single_partition_test.yml | 55 ++++++
harry-integration-external/pom.xml | 5 +
.../model/sut/external/ExternalClusterSut.java | 14 +-
harry-integration/pom.xml | 14 +-
.../src/harry/model/sut/ExternalClusterSut.java | 187 ---------------------
.../src/harry/model/sut/InJvmSutBase.java | 3 +-
.../src/harry/runner/HarryRunnerJvm.java | 43 +++++
harry-integration/test/conf/cassandra.yaml | 44 +++++
.../test/harry/ddl/SchemaGenTest.java | 15 +-
.../generators/DataGeneratorsIntegrationTest.java | 104 ++++++++++++
.../test/harry/model/IntegrationTestBase.java | 6 +-
.../model/QuiescentCheckerIntegrationTest.java | 3 +-
.../test/harry/model/TestEveryClustering.java | 89 ++++++++++
.../test/harry/op/RowVisitorTest.java | 8 +-
.../test/resources/single_partition_test.yml | 55 ++++++
15 files changed, 435 insertions(+), 210 deletions(-)
diff --git a/harry-integration-backup/test/resources/single_partition_test.yml b/harry-integration-backup/test/resources/single_partition_test.yml
new file mode 100644
index 0000000..0ebe2aa
--- /dev/null
+++ b/harry-integration-backup/test/resources/single_partition_test.yml
@@ -0,0 +1,55 @@
+seed: 1
+
+# Default schema provider generates random schema
+schema_provider:
+ default: {}
+
+drop_schema: false
+create_schema: true
+truncate_table: false
+
+clock:
+ offset:
+ offset: 1000
+
+run_time: 10
+run_time_unit: "MINUTES"
+
+system_under_test:
+ println: {}
+
+partition_descriptor_selector:
+ always_same:
+ pd: 12345
+
+clustering_descriptor_selector:
+ default:
+ modifications_per_lts:
+ type: "constant"
+ constant: 2
+ rows_per_modification:
+ type: "constant"
+ constant: 2
+ operation_kind_weights:
+ DELETE_RANGE: 1
+ DELETE_SLICE: 1
+ DELETE_ROW: 1
+ DELETE_COLUMN: 1
+ DELETE_PARTITION: 1
+ DELETE_COLUMN_WITH_STATICS: 1
+ INSERT_WITH_STATICS: 24
+ INSERT: 24
+ UPDATE_WITH_STATICS: 23
+ UPDATE: 23
+ column_mask_bitsets: null
+ max_partition_size: 100
+
+data_tracker:
+ no_op: {}
+
+runner:
+ sequential:
+ partition_visitors: []
+
+metric_reporter:
+ no_op: {}
\ No newline at end of file
diff --git a/harry-integration-external/pom.xml b/harry-integration-external/pom.xml
index 8c57e0e..0b49f6d 100755
--- a/harry-integration-external/pom.xml
+++ b/harry-integration-external/pom.xml
@@ -34,6 +34,11 @@
<dependencies>
<dependency>
+ <groupId>com.datastax.cassandra</groupId>
+ <artifactId>cassandra-driver-core</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.cassandra</groupId>
<artifactId>harry-core</artifactId>
<version>${project.parent.version}</version>
diff --git a/harry-integration-external/src/harry/model/sut/external/ExternalClusterSut.java b/harry-integration-external/src/harry/model/sut/external/ExternalClusterSut.java
index e471ede..76a4e3c 100644
--- a/harry-integration-external/src/harry/model/sut/external/ExternalClusterSut.java
+++ b/harry-integration-external/src/harry/model/sut/external/ExternalClusterSut.java
@@ -93,9 +93,7 @@ public class ExternalClusterSut implements SystemUnderTest
{
try
{
- Statement st = new SimpleStatement(statement, bindings);
- st.setConsistencyLevel(toDriverCl(cl));
- return resultSetToObjectArray(session.execute(st));
+ return resultSetToObjectArray(session.execute(statement, bindings));
}
catch (Throwable t)
{
@@ -134,9 +132,7 @@ public class ExternalClusterSut implements SystemUnderTest
public CompletableFuture<Object[][]> executeAsync(String statement, ConsistencyLevel cl, Object... bindings)
{
CompletableFuture<Object[][]> future = new CompletableFuture<>();
- Statement st = new SimpleStatement(statement, bindings);
- st.setConsistencyLevel(toDriverCl(cl));
- Futures.addCallback(session.executeAsync(st),
+ Futures.addCallback(session.executeAsync(statement, bindings),
new FutureCallback<ResultSet>()
{
public void onSuccess(ResultSet rows)
@@ -185,8 +181,10 @@ public class ExternalClusterSut implements SystemUnderTest
{
switch (cl)
{
- case ALL: return com.datastax.driver.core.ConsistencyLevel.ALL;
- case QUORUM: return com.datastax.driver.core.ConsistencyLevel.QUORUM;
+ case ALL:
+ return com.datastax.driver.core.ConsistencyLevel.ALL;
+ case QUORUM:
+ return com.datastax.driver.core.ConsistencyLevel.QUORUM;
}
throw new IllegalArgumentException("Don't know a CL: " + cl);
}
diff --git a/harry-integration/pom.xml b/harry-integration/pom.xml
index 4695d38..0b2d131 100755
--- a/harry-integration/pom.xml
+++ b/harry-integration/pom.xml
@@ -34,9 +34,14 @@
<dependencies>
<dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.cassandra</groupId>
<artifactId>harry-core</artifactId>
- <version>${project.parent.version}</version>
+ <version>${project.parent.version}</version>
</dependency>
<dependency>
@@ -62,5 +67,12 @@
<scope>test</scope>
</dependency>
</dependencies>
+ <build>
+ <testResources>
+ <testResource>
+ <directory>test/resources</directory>
+ </testResource>
+ </testResources>
+ </build>
</project>
diff --git a/harry-integration/src/harry/model/sut/ExternalClusterSut.java b/harry-integration/src/harry/model/sut/ExternalClusterSut.java
deleted file mode 100644
index 74085e4..0000000
--- a/harry-integration/src/harry/model/sut/ExternalClusterSut.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package harry.model.sut;
-
-import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.util.concurrent.FutureCallback;
-import com.google.common.util.concurrent.Futures;
-
-import com.datastax.driver.core.Cluster;
-import com.datastax.driver.core.ColumnDefinitions;
-import com.datastax.driver.core.QueryOptions;
-import com.datastax.driver.core.ResultSet;
-import com.datastax.driver.core.Row;
-import com.datastax.driver.core.Session;
-import com.datastax.driver.core.SimpleStatement;
-import com.datastax.driver.core.Statement;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import harry.core.Configuration;
-
-public class ExternalClusterSut implements SystemUnderTest
-{
- public static void init()
- {
- Configuration.registerSubtypes(ExternalClusterSutConfiguration.class);
- }
-
- private final Session session;
- private final ExecutorService executor;
-
- // TODO: pass cluster, not session
- public ExternalClusterSut(Session session)
- {
- this(session, 10);
- }
-
- public ExternalClusterSut(Session session, int threads)
- {
- this.session = session;
- this.executor = Executors.newFixedThreadPool(threads);
- }
-
- public static ExternalClusterSut create()
- {
- return new ExternalClusterSut(Cluster.builder()
- .withQueryOptions(new QueryOptions().setConsistencyLevel(toDriverCl(ConsistencyLevel.QUORUM)))
- .addContactPoints("127.0.0.1")
- .build()
- .connect());
- }
-
- public boolean isShutdown()
- {
- return session.isClosed();
- }
-
- public void shutdown()
- {
- session.close();
- executor.shutdown();
- try
- {
- executor.awaitTermination(60, TimeUnit.SECONDS);
- }
- catch (InterruptedException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- // TODO: this is rather simplistic
- public Object[][] execute(String statement, ConsistencyLevel cl, Object... bindings)
- {
- int repeat = 10;
- while (true)
- {
- try
- {
- Statement st = new SimpleStatement(statement, bindings);
- st.setConsistencyLevel(toDriverCl(cl));
- return resultSetToObjectArray(session.execute(st));
- }
- catch (Throwable t)
- {
- if (repeat < 0)
- throw t;
-
- t.printStackTrace();
- repeat--;
- // retry unconditionally
- }
- }
- }
-
- public static Object[][] resultSetToObjectArray(ResultSet rs)
- {
- List<Row> rows = rs.all();
- if (rows.size() == 0)
- return new Object[0][];
- Object[][] results = new Object[rows.size()][];
- for (int i = 0; i < results.length; i++)
- {
- Row row = rows.get(i);
- ColumnDefinitions cds = row.getColumnDefinitions();
- Object[] result = new Object[cds.size()];
- for (int j = 0; j < cds.size(); j++)
- {
- if (!row.isNull(j))
- result[j] = row.getObject(j);
- }
- results[i] = result;
- }
- return results;
- }
-
- public CompletableFuture<Object[][]> executeAsync(String statement, ConsistencyLevel cl, Object... bindings)
- {
- CompletableFuture<Object[][]> future = new CompletableFuture<>();
- Statement st = new SimpleStatement(statement, bindings);
- st.setConsistencyLevel(toDriverCl(cl));
- Futures.addCallback(session.executeAsync(st),
- new FutureCallback<ResultSet>()
- {
- public void onSuccess(ResultSet rows)
- {
- future.complete(resultSetToObjectArray(rows));
- }
-
- public void onFailure(Throwable throwable)
- {
- future.completeExceptionally(throwable);
- }
- },
- executor);
-
- return future;
- }
-
- public static com.datastax.driver.core.ConsistencyLevel toDriverCl(SystemUnderTest.ConsistencyLevel cl)
- {
- switch (cl)
- {
- case ALL: return com.datastax.driver.core.ConsistencyLevel.ALL;
- case QUORUM: return com.datastax.driver.core.ConsistencyLevel.QUORUM;
- }
- throw new IllegalArgumentException("Don't know a CL: " + cl);
- }
-
- @JsonTypeName("external")
- public static class ExternalClusterSutConfiguration implements Configuration.SutConfiguration
- {
- public final String[] hosts;
-
- public ExternalClusterSutConfiguration(@JsonProperty(value = "hosts") String[] hosts)
- {
- this.hosts = hosts;
- }
-
- public SystemUnderTest make()
- {
- Cluster cluster = Cluster.builder().addContactPoints(hosts).build();
- Session session = cluster.newSession().init();
- return new ExternalClusterSut(session);
- }
- }
-}
\ No newline at end of file
diff --git a/harry-integration/src/harry/model/sut/InJvmSutBase.java b/harry-integration/src/harry/model/sut/InJvmSutBase.java
index e29481f..aa2608f 100644
--- a/harry-integration/src/harry/model/sut/InJvmSutBase.java
+++ b/harry-integration/src/harry/model/sut/InJvmSutBase.java
@@ -30,20 +30,19 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
-import com.google.common.collect.Iterators;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import harry.core.Configuration;
-import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.distributed.api.ICluster;
import org.apache.cassandra.distributed.api.IInstance;
import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.distributed.api.IMessage;
import org.apache.cassandra.distributed.api.IMessageFilters;
+import relocated.shaded.com.google.common.collect.Iterators;
public class InJvmSutBase<NODE extends IInstance, CLUSTER extends ICluster<NODE>> implements SystemUnderTest.FaultInjectingSut
{
diff --git a/harry-integration/src/harry/runner/HarryRunnerJvm.java b/harry-integration/src/harry/runner/HarryRunnerJvm.java
new file mode 100644
index 0000000..fbb30b9
--- /dev/null
+++ b/harry-integration/src/harry/runner/HarryRunnerJvm.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.runner;
+
+import harry.core.Configuration;
+import harry.model.sut.InJvmSut;
+
+import java.io.File;
+
+public class HarryRunnerJvm extends HarryRunner {
+
+ public static void main(String[] args) throws Throwable {
+ InJvmSut.init();
+
+ HarryRunnerJvm runner = new HarryRunnerJvm();
+ File configFile = runner.loadConfig(args);
+
+ Configuration configuration = Configuration.fromFile(configFile);
+ runner.run(configuration);
+ }
+
+
+ @Override
+ public void beforeRun(Runner runner) {
+
+ }
+}
diff --git a/harry-integration/test/conf/cassandra.yaml b/harry-integration/test/conf/cassandra.yaml
new file mode 100644
index 0000000..2536aa8
--- /dev/null
+++ b/harry-integration/test/conf/cassandra.yaml
@@ -0,0 +1,44 @@
+#
+# Warning!
+# Consider the effects on 'o.a.c.i.s.LegacySSTableTest' before changing schemas in this file.
+#
+cluster_name: Test Cluster
+memtable_allocation_type: heap_buffers
+commitlog_sync: batch
+commitlog_sync_batch_window_in_ms: 1.0
+commitlog_segment_size_in_mb: 5
+commitlog_directory: build/test/cassandra/commitlog
+hints_directory: build/test/cassandra/hints
+partitioner: org.apache.cassandra.dht.ByteOrderedPartitioner
+listen_address: 127.0.0.1
+storage_port: 7010
+rpc_port: 9170
+start_native_transport: true
+native_transport_port: 9042
+column_index_size_in_kb: 4
+saved_caches_directory: build/test/cassandra/saved_caches
+data_file_directories:
+ - build/test/cassandra/data
+disk_access_mode: mmap
+seed_provider:
+ - class_name: org.apache.cassandra.locator.SimpleSeedProvider
+ parameters:
+ - seeds: "127.0.0.1"
+endpoint_snitch: org.apache.cassandra.locator.SimpleSnitch
+dynamic_snitch: true
+request_scheduler: org.apache.cassandra.scheduler.RoundRobinScheduler
+request_scheduler_id: keyspace
+server_encryption_options:
+ internode_encryption: none
+ keystore: conf/.keystore
+ keystore_password: cassandra
+ truststore: conf/.truststore
+ truststore_password: cassandra
+incremental_backups: true
+concurrent_compactors: 4
+compaction_throughput_mb_per_sec: 0
+row_cache_class_name: org.apache.cassandra.cache.OHCProvider
+row_cache_size_in_mb: 16
+enable_user_defined_functions: true
+enable_scripted_user_defined_functions: true
+enable_drop_compact_storage: true
diff --git a/harry-integration/test/harry/ddl/SchemaGenTest.java b/harry-integration/test/harry/ddl/SchemaGenTest.java
index 74a9c53..2f7e898 100644
--- a/harry-integration/test/harry/ddl/SchemaGenTest.java
+++ b/harry-integration/test/harry/ddl/SchemaGenTest.java
@@ -50,11 +50,12 @@ public class SchemaGenTest extends CQLTester
// TODO: compact storage tests
@Test
- public void testSelectForwardAndReverseIteration()
+ public void testSelectForwardAndReverseIteration() throws Throwable
{
Generator<SchemaSpec> gen = new SchemaGenerators.Builder(KEYSPACE).partitionKeyColumnCount(1, 4)
.clusteringColumnCount(1, 10)
.regularColumnCount(0, 10)
+ .staticColumnCount(0, 10)
.generator();
@@ -78,11 +79,12 @@ public class SchemaGenTest extends CQLTester
}
@Test
- public void createTableRoundTrip()
+ public void createTableRoundTrip() throws Throwable
{
Generator<SchemaSpec> gen = new SchemaGenerators.Builder(KEYSPACE).partitionKeyColumnCount(1, 10)
- .clusteringColumnCount(0, 10)
+ .clusteringColumnCount(1, 10)
.regularColumnCount(0, 10)
+ .staticColumnCount(0, 10)
.generator();
TestRunner.test(gen,
@@ -93,6 +95,7 @@ public class SchemaGenTest extends CQLTester
compareColumns(schemaDefinition.partitionKeys, tableMetadata.partitionKeyColumns());
compareColumns(schemaDefinition.clusteringKeys, tableMetadata.clusteringColumns());
compareColumns(schemaDefinition.regularColumns, tableMetadata.regularColumns());
+ compareColumns(schemaDefinition.staticColumns, tableMetadata.staticColumns());
});
}
@@ -108,8 +111,8 @@ public class SchemaGenTest extends CQLTester
ColumnSpec.regularColumn("v2", ColumnSpec.asciiType),
ColumnSpec.regularColumn("v3", ColumnSpec.int64Type),
ColumnSpec.regularColumn("v4", ColumnSpec.int64Type)),
- Arrays.asList(ColumnSpec.staticColumn("regular1", ColumnSpec.asciiType),
- ColumnSpec.staticColumn("regular2", ColumnSpec.int64Type)));
+ Arrays.asList(ColumnSpec.staticColumn("static1", ColumnSpec.asciiType),
+ ColumnSpec.staticColumn("static2", ColumnSpec.int64Type)));
String tableDef = spec.compile().cql();
@@ -118,6 +121,7 @@ public class SchemaGenTest extends CQLTester
compareColumns(spec.partitionKeys, tableMetadata.partitionKeyColumns());
compareColumns(spec.clusteringKeys, tableMetadata.clusteringColumns());
compareColumns(spec.regularColumns, tableMetadata.regularColumns());
+ compareColumns(spec.staticColumns, tableMetadata.staticColumns());
}
@@ -126,6 +130,7 @@ public class SchemaGenTest extends CQLTester
{
Gen<Pair<Integer, Integer>> ckCounts = integers().between(0, 4).zip(integers().between(0, 6), Pair::create);
Gen<Pair<Integer, Integer>> regCounts = integers().between(0, 4).zip(integers().between(0, 6), Pair::create);
+// Gen<Pair<Integer, Integer>> staticCounts = integers().between(0, 4).zip(integers().between(0, 6), Pair::create);
Gen<Pair<Integer, Integer>> pkCounts = integers().between(1, 4).zip(integers().between(0, 6), Pair::create);
Gen<SchemaGenerationInputs> inputs = pkCounts.zip(ckCounts, regCounts,
diff --git a/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java b/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
index affd954..cadee0c 100644
--- a/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
+++ b/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
@@ -19,11 +19,28 @@
package harry.generators;
import java.util.Random;
+import java.util.concurrent.CompletableFuture;
import org.junit.Test;
+import harry.core.Configuration;
+import harry.core.Run;
import harry.ddl.ColumnSpec;
+import harry.ddl.SchemaGenerators;
+import harry.ddl.SchemaSpec;
+import harry.generators.distribution.Distribution;
+import harry.model.NoOpChecker;
+import harry.model.OpSelectors;
+import harry.model.sut.SystemUnderTest;
+import harry.runner.MutatingPartitionVisitor;
+import harry.runner.MutatingRowVisitor;
+import harry.runner.PartitionVisitor;
+import harry.runner.SinglePartitionValidator;
+import harry.util.TestRunner;
import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.distributed.impl.RowUtil;
+import relocated.shaded.com.google.common.collect.Iterators;
public class DataGeneratorsIntegrationTest extends CQLTester
{
@@ -60,4 +77,91 @@ public class DataGeneratorsIntegrationTest extends CQLTester
}
}
}
+
+ @Test
+ public void queryParseabilityTest() throws Throwable
+ {
+ Generator<SchemaSpec> gen = new SchemaGenerators.Builder(KEYSPACE).partitionKeyColumnCount(2, 4)
+ .clusteringColumnCount(1, 4)
+ .regularColumnCount(1, 4)
+ .staticColumnCount(1, 4)
+ .generator();
+
+ TestRunner.test(gen,
+ (schema) -> {
+ createTable(schema.compile().cql());
+
+ Configuration.ConfigurationBuilder builder = Configuration.fromFile(getClass().getClassLoader().getResource("single_partition_test.yml").getFile())
+ .unbuild()
+ .setSchemaProvider(new Configuration.FixedSchemaProviderConfiguration(schema, null, null, null, null))
+ .setSUT(CqlTesterSut::new);
+
+ for (OpSelectors.OperationKind opKind : OpSelectors.OperationKind.values())
+ {
+ Run run = builder
+ .setClusteringDescriptorSelector((rng, schema_) -> {
+ return new OpSelectors.DefaultDescriptorSelector(rng,
+ OpSelectors.columnSelectorBuilder().forAll(schema_).build(),
+ OpSelectors.OperationSelector.weighted(Surjections.weights(100), opKind),
+ new Distribution.ConstantDistribution(2),
+ new Distribution.ConstantDistribution(2),
+ 100);
+ })
+ .build()
+ .createRun();
+
+ PartitionVisitor visitor = new MutatingPartitionVisitor(run, MutatingRowVisitor::new);
+ for (int lts = 0; lts < 100; lts++)
+ visitor.visitPartition(lts);
+ }
+
+ Run run = builder.build()
+ .createRun();
+ PartitionVisitor visitor = new SinglePartitionValidator(100, run, NoOpChecker::new);
+ for (int lts = 0; lts < 100; lts++)
+ visitor.visitPartition(lts);
+
+ });
+
+ }
+
+ public class CqlTesterSut implements SystemUnderTest
+ {
+ public boolean isShutdown()
+ {
+ return false;
+ }
+
+ public void shutdown()
+ {
+ cleanup();
+ }
+
+ public void schemaChange(String statement)
+ {
+ createTable(statement);
+ }
+
+ public Object[][] execute(String statement, ConsistencyLevel cl, Object... bindings)
+ {
+ try
+ {
+ UntypedResultSet res = DataGeneratorsIntegrationTest.this.execute(statement, bindings);
+ if (res == null)
+ return new Object[][] {};
+
+ return Iterators.toArray(RowUtil.toIter(res), Object[].class);
+ }
+ catch (Throwable throwable)
+ {
+ throw new RuntimeException(throwable);
+ }
+ }
+
+ public CompletableFuture<Object[][]> executeAsync(String statement, ConsistencyLevel cl, Object... bindings)
+ {
+ return CompletableFuture.completedFuture(execute(statement, cl, bindings));
+ }
+ }
}
+
diff --git a/harry-integration/test/harry/model/IntegrationTestBase.java b/harry-integration/test/harry/model/IntegrationTestBase.java
index 551124f..a6a9697 100644
--- a/harry-integration/test/harry/model/IntegrationTestBase.java
+++ b/harry-integration/test/harry/model/IntegrationTestBase.java
@@ -81,8 +81,10 @@ public class IntegrationTestBase extends TestBaseImpl
.addWeight(OpSelectors.OperationKind.DELETE_SLICE, 1)
.addWeight(OpSelectors.OperationKind.DELETE_PARTITION, 1)
.addWeight(OpSelectors.OperationKind.DELETE_COLUMN_WITH_STATICS, 5)
- .addWeight(OpSelectors.OperationKind.WRITE_WITH_STATICS, 45)
- .addWeight(OpSelectors.OperationKind.WRITE, 45)
+ .addWeight(OpSelectors.OperationKind.INSERT_WITH_STATICS, 20)
+ .addWeight(OpSelectors.OperationKind.INSERT, 20)
+ .addWeight(OpSelectors.OperationKind.UPDATE_WITH_STATICS, 25)
+ .addWeight(OpSelectors.OperationKind.UPDATE, 25)
.build());
}
diff --git a/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java b/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
index 4c83276..7d5956f 100644
--- a/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
+++ b/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
@@ -128,7 +128,8 @@ public class QuiescentCheckerIntegrationTest extends ModelTestBase
(t, run) -> {
String expected = "doesn't match the one predicted by the model";
String expected2 = "don't match ones predicted by the model";
- if (t.getMessage().contains(expected) || t.getMessage().contains(expected2))
+ String expected3 = "Found a row in the model that is not present in the resultset";
+ if (t.getMessage().contains(expected) || t.getMessage().contains(expected2) || t.getMessage().contains(expected3))
return;
throw new AssertionError(String.format("Exception string mismatch.\nExpected error: %s.\nActual error: %s", expected, t.getMessage()),
diff --git a/harry-integration/test/harry/model/TestEveryClustering.java b/harry-integration/test/harry/model/TestEveryClustering.java
new file mode 100644
index 0000000..f844b41
--- /dev/null
+++ b/harry-integration/test/harry/model/TestEveryClustering.java
@@ -0,0 +1,89 @@
+package harry.model;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.function.Supplier;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import harry.core.Configuration;
+import harry.core.Run;
+import harry.ddl.SchemaGenerators;
+import harry.ddl.SchemaSpec;
+import harry.generators.distribution.Distribution;
+import harry.model.sut.SystemUnderTest;
+import harry.operations.CompiledStatement;
+import harry.operations.Relation;
+import harry.runner.FaultInjectingPartitionVisitor;
+import harry.runner.LoggingPartitionVisitor;
+import harry.runner.MutatingPartitionVisitor;
+import harry.runner.MutatingRowVisitor;
+import harry.runner.PartitionVisitor;
+import harry.runner.Query;
+import harry.runner.QueryGenerator;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+
+public class TestEveryClustering extends IntegrationTestBase
+{
+ int CYCLES = 1000;
+
+ @Test
+ public void basicQuerySelectorTest()
+ {
+ Supplier<SchemaSpec> schemaGen = SchemaGenerators.progression(SchemaGenerators.DEFAULT_SWITCH_AFTER);
+ for (int cnt = 0; cnt < Integer.MAX_VALUE; cnt++)
+ {
+ beforeEach();
+ SchemaSpec schemaSpec = schemaGen.get();
+
+ System.out.println(schemaSpec.compile().cql());
+ int partitionSize = 1000;
+
+ Configuration config = sharedConfiguration(cnt, schemaSpec)
+ .setPartitionDescriptorSelector(new Configuration.DefaultPDSelectorConfiguration(1, partitionSize))
+ .setClusteringDescriptorSelector(sharedCDSelectorConfiguration()
+ .setNumberOfModificationsDistribution(() -> new Distribution.ConstantDistribution(1L))
+ .setRowsPerModificationDistribution(() -> new Distribution.ConstantDistribution(1L))
+ .setMaxPartitionSize(250)
+ .build())
+ .build();
+
+ Run run = config.createRun();
+ run.sut.schemaChange(run.schemaSpec.compile().cql());
+ OpSelectors.MonotonicClock clock = run.clock;
+
+ Set<Long> visitedCds = new HashSet<>();
+ PartitionVisitor partitionVisitor = new LoggingPartitionVisitor(run, (r) -> {
+ return new MutatingRowVisitor(r) {
+ public CompiledStatement perform(OpSelectors.OperationKind op, long lts, long pd, long cd, long opId)
+ {
+ visitedCds.add(cd);
+ return super.perform(op, lts, pd, cd, opId);
+ }
+ };
+ });
+ sut.cluster().stream().forEach((IInvokableInstance node) -> node.nodetool("disableautocompaction"));
+ for (int i = 0; i < CYCLES; i++)
+ {
+ long lts = clock.nextLts();
+ partitionVisitor.visitPartition(lts);
+
+ if (i > 0 && i % 250 == 0)
+ sut.cluster().stream().forEach((IInvokableInstance node) -> node.nodetool("flush", schemaSpec.keyspace, schemaSpec.table));
+ }
+
+ for (Long cd : visitedCds)
+ {
+ Query query = new Query.SingleClusteringQuery(Query.QueryKind.SINGLE_CLUSTERING,
+ run.pdSelector.pd(0),
+ cd,
+ false,
+ Relation.eqRelations(run.schemaSpec.ckGenerator.slice(cd), run.schemaSpec.clusteringKeys),
+ run.schemaSpec);
+ Model model = new QuiescentChecker(run);
+ model.validate(query);
+ }
+ }
+ }
+}
diff --git a/harry-integration/test/harry/op/RowVisitorTest.java b/harry-integration/test/harry/op/RowVisitorTest.java
index d7061f9..ae51115 100644
--- a/harry-integration/test/harry/op/RowVisitorTest.java
+++ b/harry-integration/test/harry/op/RowVisitorTest.java
@@ -83,10 +83,10 @@ public class RowVisitorTest extends CQLTester
MutatingRowVisitor visitor = new MutatingRowVisitor(run);
long[] descriptors = rand.next(4);
- execute(visitor.write(Math.abs(descriptors[0]),
- descriptors[1],
- descriptors[2],
- descriptors[3]));
+ execute(visitor.insert(Math.abs(descriptors[0]),
+ descriptors[1],
+ descriptors[2],
+ descriptors[3]));
}
}
diff --git a/harry-integration/test/resources/single_partition_test.yml b/harry-integration/test/resources/single_partition_test.yml
new file mode 100644
index 0000000..0ebe2aa
--- /dev/null
+++ b/harry-integration/test/resources/single_partition_test.yml
@@ -0,0 +1,55 @@
+seed: 1
+
+# Default schema provider generates random schema
+schema_provider:
+ default: {}
+
+drop_schema: false
+create_schema: true
+truncate_table: false
+
+clock:
+ offset:
+ offset: 1000
+
+run_time: 10
+run_time_unit: "MINUTES"
+
+system_under_test:
+ println: {}
+
+partition_descriptor_selector:
+ always_same:
+ pd: 12345
+
+clustering_descriptor_selector:
+ default:
+ modifications_per_lts:
+ type: "constant"
+ constant: 2
+ rows_per_modification:
+ type: "constant"
+ constant: 2
+ operation_kind_weights:
+ DELETE_RANGE: 1
+ DELETE_SLICE: 1
+ DELETE_ROW: 1
+ DELETE_COLUMN: 1
+ DELETE_PARTITION: 1
+ DELETE_COLUMN_WITH_STATICS: 1
+ INSERT_WITH_STATICS: 24
+ INSERT: 24
+ UPDATE_WITH_STATICS: 23
+ UPDATE: 23
+ column_mask_bitsets: null
+ max_partition_size: 100
+
+data_tracker:
+ no_op: {}
+
+runner:
+ sequential:
+ partition_visitors: []
+
+metric_reporter:
+ no_op: {}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org