You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by if...@apache.org on 2021/10/01 14:21:31 UTC
[cassandra-harry] branch trunk updated (ddd643e -> f6b4df6)
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-harry.git.
from ddd643e Fix MetricReporter serialization; Update example yaml configs for required props.
new df40fa3 Core improvements
new 469533e Integration improvements
new 59604db Adjust config files
new 83dd5a6 Move classes to appropriate packages
new f6b4df6 Add history builder and an ability to write unit-tests with Harry
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
README.md | 221 +++++++-
conf/{example.yaml => default.yaml} | 56 +-
conf/external.yaml | 56 +-
docker/Dockerfile.local | 2 +-
docker/run.sh | 2 +-
harry-core/pom.xml | 9 +-
harry-core/src/harry/core/Configuration.java | 128 +++--
harry-core/src/harry/core/Run.java | 2 +-
harry-core/src/harry/core/VisibleForTesting.java | 5 +
.../src/harry/corruptor/AddExtraRowCorruptor.java | 4 +-
.../harry/corruptor/QueryResponseCorruptor.java | 3 +-
harry-core/src/harry/corruptor/RowCorruptor.java | 2 +-
harry-core/src/harry/ddl/ColumnSpec.java | 10 +-
harry-core/src/harry/ddl/SchemaGenerators.java | 76 ++-
harry-core/src/harry/ddl/SchemaSpec.java | 102 +++-
harry-core/src/harry/dsl/HistoryBuilder.java | 625 +++++++++++++++++++++
.../src/harry/generators/DataGenerators.java | 17 +-
.../src/harry/generators/RandomGenerator.java | 3 +-
harry-core/src/harry/generators/RngUtils.java | 6 +-
harry-core/src/harry/generators/Surjections.java | 2 +-
.../harry/model/AlwaysSamePartitionSelector.java | 69 +++
harry-core/src/harry/model/Model.java | 3 +-
harry-core/src/harry/model/NoOpChecker.java | 5 +-
harry-core/src/harry/model/OpSelectors.java | 67 ++-
harry-core/src/harry/model/QuiescentChecker.java | 18 +-
harry-core/src/harry/model/SelectHelper.java | 116 ++--
.../model/clock/ApproximateMonotonicClock.java | 8 +-
harry-core/src/harry/model/clock/OffsetClock.java | 25 +
harry-core/src/harry/model/sut/PrintlnSut.java | 18 +
harry-core/src/harry/operations/DeleteHelper.java | 61 +-
.../src/harry/{runner => operations}/Query.java | 5 +-
.../{runner => operations}/QueryGenerator.java | 6 +-
harry-core/src/harry/operations/Relation.java | 80 +--
harry-core/src/harry/operations/WriteHelper.java | 171 +++---
harry-core/src/harry/reconciler/Reconciler.java | 145 +++--
harry-core/src/harry/runner/DataTracker.java | 1 +
.../src/harry/runner/DefaultDataTracker.java | 7 +-
.../src/harry/runner/LoggingPartitionVisitor.java | 79 ---
.../src/harry/runner/MutatingPartitionVisitor.java | 144 -----
harry-core/src/harry/runner/Runner.java | 46 +-
harry-core/src/harry/util/BitSet.java | 2 +-
harry-core/src/harry/util/TestRunner.java | 48 +-
.../AllPartitionsValidator.java | 7 +-
.../CorruptingVisitor.java} | 13 +-
.../DelegatingVisitor.java} | 38 +-
.../GeneratingVisitor.java} | 60 +-
harry-core/src/harry/visitors/LoggingVisitor.java | 90 +++
.../{runner => visitors}/MutatingRowVisitor.java | 60 +-
harry-core/src/harry/visitors/MutatingVisitor.java | 169 ++++++
.../OperationExecutor.java} | 26 +-
.../ParallelRecentValidator.java} | 30 +-
.../{runner => visitors}/ParallelValidator.java | 6 +-
.../RecentValidator.java} | 22 +-
.../src/harry/visitors/ReplayingVisitor.java | 121 ++++
.../src/harry/{runner => visitors}/Sampler.java | 10 +-
.../SingleValidator.java} | 14 +-
.../VisitExecutor.java} | 31 +-
.../Visitor.java} | 14 +-
harry-core/test/harry/model/OpSelectorsTest.java | 46 +-
harry-core/test/harry/operations/RelationTest.java | 4 +-
.../dependency-reduced-pom.xml | 54 ++
harry-integration-external/pom.xml | 5 +
.../model/sut/external/ExternalClusterSut.java | 14 +-
harry-integration/pom.xml | 14 +-
.../src/harry/model/sut/ExternalClusterSut.java | 187 ------
.../model/sut/InJVMTokenAwareVisitExecutor.java | 114 ++++
.../src/harry/model/sut/InJvmSut.java | 22 +
.../src/harry/model/sut/InJvmSutBase.java | 3 +-
...tionVisitor.java => FaultInjectingVisitor.java} | 22 +-
.../src/harry/runner/HarryRunnerJvm.java | 13 +-
.../src/harry/runner/QueryingNoOpChecker.java | 1 +
.../harry/runner/RepairingLocalStateValidator.java | 21 +-
harry-integration/src/harry/runner/Reproduce.java | 1 +
.../src/harry/runner/TrivialShrinker.java | 76 +--
.../src/harry/visitors/SkippingVisitor.java | 52 +-
.../test}/conf/cassandra.yaml | 39 +-
.../test/harry/ddl/SchemaGenTest.java | 17 +-
.../generators/DataGeneratorsIntegrationTest.java | 122 +++-
.../harry/model/HistoryBuilderIntegrationTest.java | 197 +++++++
.../test/harry/model/HistoryBuilderTest.java | 186 ++++++
.../harry/model/InJVMTokenAwareExecutorTest.java | 91 +++
.../test/harry/model/IntegrationTestBase.java | 6 +-
.../test/harry/model/ModelTestBase.java | 24 +-
.../harry/model/QuerySelectorNegativeTest.java | 14 +-
.../test/harry/model/QuerySelectorTest.java | 18 +-
.../model/QuiescentCheckerIntegrationTest.java | 13 +-
.../test/harry/op/RowVisitorTest.java | 10 +-
.../test/resources/single_partition_test.yml | 55 ++
pom.xml | 24 +-
run-jvm.sh | 2 +-
90 files changed, 3320 insertions(+), 1313 deletions(-)
rename conf/{example.yaml => default.yaml} (77%)
create mode 100644 harry-core/src/harry/core/VisibleForTesting.java
create mode 100644 harry-core/src/harry/dsl/HistoryBuilder.java
create mode 100644 harry-core/src/harry/model/AlwaysSamePartitionSelector.java
rename harry-core/src/harry/{runner => operations}/Query.java (98%)
rename harry-core/src/harry/{runner => operations}/QueryGenerator.java (98%)
delete mode 100644 harry-core/src/harry/runner/LoggingPartitionVisitor.java
delete mode 100644 harry-core/src/harry/runner/MutatingPartitionVisitor.java
rename harry-core/src/harry/{runner => visitors}/AllPartitionsValidator.java (97%)
rename harry-core/src/harry/{runner/CorruptingPartitionVisitor.java => visitors/CorruptingVisitor.java} (93%)
copy harry-core/src/harry/{model/clock/OffsetClock.java => visitors/DelegatingVisitor.java} (52%)
rename harry-core/src/harry/{runner/AbstractPartitionVisitor.java => visitors/GeneratingVisitor.java} (55%)
create mode 100644 harry-core/src/harry/visitors/LoggingVisitor.java
rename harry-core/src/harry/{runner => visitors}/MutatingRowVisitor.java (58%)
create mode 100644 harry-core/src/harry/visitors/MutatingVisitor.java
rename harry-core/src/harry/{runner/Operation.java => visitors/OperationExecutor.java} (75%)
rename harry-core/src/harry/{runner/ParallelRecentPartitionValidator.java => visitors/ParallelRecentValidator.java} (79%)
rename harry-core/src/harry/{runner => visitors}/ParallelValidator.java (97%)
rename harry-core/src/harry/{runner/RecentPartitionValidator.java => visitors/RecentValidator.java} (89%)
create mode 100644 harry-core/src/harry/visitors/ReplayingVisitor.java
rename harry-core/src/harry/{runner => visitors}/Sampler.java (95%)
rename harry-core/src/harry/{runner/SinglePartitionValidator.java => visitors/SingleValidator.java} (85%)
copy harry-core/src/harry/{model/NoOpChecker.java => visitors/VisitExecutor.java} (64%)
rename harry-core/src/harry/{runner/PartitionVisitor.java => visitors/Visitor.java} (78%)
create mode 100644 harry-integration-external/dependency-reduced-pom.xml
delete mode 100644 harry-integration/src/harry/model/sut/ExternalClusterSut.java
create mode 100644 harry-integration/src/harry/model/sut/InJVMTokenAwareVisitExecutor.java
rename harry-integration/src/harry/runner/{FaultInjectingPartitionVisitor.java => FaultInjectingVisitor.java} (76%)
copy harry-integration-external/src/harry/runner/external/HarryRunnerExternal.java => harry-integration/src/harry/runner/HarryRunnerJvm.java (79%)
copy harry-core/src/harry/model/clock/OffsetClock.java => harry-integration/src/harry/visitors/SkippingVisitor.java (51%)
copy {test => harry-integration/test}/conf/cassandra.yaml (55%)
create mode 100644 harry-integration/test/harry/model/HistoryBuilderIntegrationTest.java
create mode 100644 harry-integration/test/harry/model/HistoryBuilderTest.java
create mode 100644 harry-integration/test/harry/model/InJVMTokenAwareExecutorTest.java
create mode 100644 harry-integration/test/resources/single_partition_test.yml
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra-harry] 05/05: Add history builder and an ability to
write unit-tests with Harry
Posted by if...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-harry.git
commit f6b4df664b5ec79cf555aa0fb34e26f40fd9e9cd
Author: Alex Petrov <ol...@gmail.com>
AuthorDate: Mon Sep 27 12:56:33 2021 +0200
Add history builder and an ability to write unit-tests with Harry
Implement a full repair test
---
README.md | 218 ++++++-
conf/default.yaml | 2 +-
conf/external.yaml | 2 +-
harry-core/src/harry/core/Configuration.java | 78 +--
.../src/harry/corruptor/AddExtraRowCorruptor.java | 2 +-
harry-core/src/harry/dsl/HistoryBuilder.java | 625 +++++++++++++++++++++
harry-core/src/harry/model/OpSelectors.java | 25 +-
harry-core/src/harry/model/QuiescentChecker.java | 14 +-
.../model/clock/ApproximateMonotonicClock.java | 5 +
harry-core/src/harry/model/clock/OffsetClock.java | 5 +
harry-core/src/harry/operations/WriteHelper.java | 3 +-
harry-core/src/harry/reconciler/Reconciler.java | 86 +--
harry-core/src/harry/runner/DataTracker.java | 2 -
.../src/harry/runner/DefaultDataTracker.java | 5 +-
harry-core/src/harry/runner/Runner.java | 46 +-
harry-core/src/harry/util/TestRunner.java | 39 ++
.../src/harry/visitors/AllPartitionsValidator.java | 4 +-
...artitionVisitor.java => CorruptingVisitor.java} | 8 +-
.../src/harry/visitors/DelegatingVisitor.java | 61 ++
...artitionVisitor.java => GeneratingVisitor.java} | 63 +--
.../harry/visitors/LoggingPartitionVisitor.java | 79 ---
harry-core/src/harry/visitors/LoggingVisitor.java | 90 +++
.../harry/visitors/MutatingPartitionVisitor.java | 145 -----
.../src/harry/visitors/MutatingRowVisitor.java | 18 +-
harry-core/src/harry/visitors/MutatingVisitor.java | 169 ++++++
.../{Operation.java => OperationExecutor.java} | 4 +-
...Validator.java => ParallelRecentValidator.java} | 26 +-
.../src/harry/visitors/ParallelValidator.java | 4 +-
...artitionValidator.java => RecentValidator.java} | 16 +-
.../src/harry/visitors/ReplayingVisitor.java | 121 ++++
harry-core/src/harry/visitors/Sampler.java | 8 +-
...artitionValidator.java => SingleValidator.java} | 11 +-
.../{PartitionVisitor.java => VisitExecutor.java} | 23 +-
.../{PartitionVisitor.java => Visitor.java} | 12 +-
harry-core/test/harry/model/OpSelectorsTest.java | 12 +-
harry-core/test/harry/operations/RelationTest.java | 2 +-
.../test/resources/single_partition_test.yml | 55 --
.../model/sut/InJVMTokenAwareVisitExecutor.java | 114 ++++
.../src/harry/model/sut/InJvmSut.java | 22 +
.../src/harry/runner/FaultInjectingVisitor.java | 22 +-
.../harry/runner/RepairingLocalStateValidator.java | 20 +-
.../src/harry/runner/TrivialShrinker.java | 29 +-
.../harry/visitors/SkippingPartitionVisitor.java | 53 --
.../src/harry/visitors/SkippingVisitor.java | 36 +-
.../generators/DataGeneratorsIntegrationTest.java | 32 +-
.../harry/model/HistoryBuilderIntegrationTest.java | 197 +++++++
.../test/harry/model/HistoryBuilderTest.java | 186 ++++++
.../harry/model/InJVMTokenAwareExecutorTest.java | 91 +++
.../test/harry/model/ModelTestBase.java | 22 +-
.../harry/model/QuerySelectorNegativeTest.java | 10 +-
.../test/harry/model/QuerySelectorTest.java | 12 +-
.../model/QuiescentCheckerIntegrationTest.java | 8 +-
.../test/harry/model/TestEveryClustering.java | 83 ---
.../test/resources/single_partition_test.yml | 2 +-
54 files changed, 2284 insertions(+), 743 deletions(-)
diff --git a/README.md b/README.md
index 841e4ea..5c45c09 100644
--- a/README.md
+++ b/README.md
@@ -1,9 +1,171 @@
# Harry, a fuzz testing tool for Apache Cassandra
-Project aims to generate _reproducible_ workloads that are as close to real-life
+The project aims to generate _reproducible_ workloads that are as close to real-life
as possible, while being able to _efficiently_ verify the cluster state against
the model without pausing the workload itself.
+# Introduction
+
+Harry has two primary modes of functionality:
+
+ * Unit test mode: in which you define specific sequences of
+ operations and let Harry test these operations using different
+ schemas and conditions.
+ * Exploratory/fuzz mode: in which you define distributions of events
+ rather rather than sequences themselves, and let Harry try out
+ different things.
+
+Usually, in unit-test mode, we’re applying several write operations to
+the cluster state and then run different read queries and validate
+their results. To learn more about writing unit tests, refer to the "Writing
+Unit Tests" section.
+
+In exploratory mode, we continuously apply write operations to the
+cluster and validate their state, allowing data size to grow and simulating
+real-life behaviour. To learn more about implementing test cases using
+fuzz mode, refer to the "Implementing Tests" section of this guide, but it's likely
+you'll have to read the rest of this document to implement more
+complex scenarios.
+
+# Writing Unit Tests
+
+To write unit tests with Harry, there's no special knowledge required.
+Usually, unit tests are written by simply hardcoding the schema and then writing
+several modification statements one after the other, and then manually validating results
+of a `SELECT` query. This might work for simple scenarios, but there’s still a chance
+that for some other schema or some combination of values the tested feature may not work.
+
+To improve the situation, we can express the test in more abstract
+terms and, instead of writing it using specific statements, we can
+describe which statement _types_ are to be used:
+
+```
+test(new SchemaGenerators.Builder("harry")
+ .partitionKeySpec(1, 5)
+ .clusteringKeySpec(1, 5)
+ .regularColumnSpec(1, 10)
+ .generator(),
+ historyBuilder -> {
+ historyBuilder.nextPartition()
+ .simultaneously()
+ .randomOrder()
+ .partitionDeletion()
+ .rangeDeletion()
+ .finish();
+ });
+```
+
+This spec can be used to generate clusters of different sizes,
+configured with different schemas, executing the given a sequence of
+actions both in isolation, and combined with other randomly generated
+ones, with failure-injection.
+
+Best of all is that this test will _not only_ ensure that such a
+sequence of actions does not produce an exception, but also would
+ensure that cluster will respond with correct results to _any_ allowed
+read query.
+
+`HistoryBuilder` is using the configuration provided by Harry `Run`, which
+can be written either using `Configuration#ConfigurationBuilder` like
+we did
+[here](https://github.com/apache/cassandra-harry/blob/ddd643ecc904258abe5e2f73d9b612793b0ac0e6/harry-integration/test/harry/model/IntegrationTestBase.java#L91),
+or provided in a [yaml file](https://github.com/apache/cassandra-harry/blob/ddd643ecc904258abe5e2f73d9b612793b0ac0e6/conf/example.yaml).
+
+To begin specifying operations for a new partition,
+`HistoryBuilder#nextPartition` has to be called, which returns a
+`PartitionBuilder`. For the commands within this partition, you have
+a choice between:
+ * `PartitionBuilder#simultaneously`, which will execute listed
+ operations with the same timestamp
+ * `PartitionBuilder#sequentially`, which will execute listed operations
+ with monotonically increasing timestamps, giving each operation its
+ own timestamp
+
+Similarly, you can choose between:
+ * `PartitionBuilder#randomOrder`, which will execute listed operations
+ in random order
+ * `PartitionBuilder#strictOrder`, which will execute listed operations
+ in the order specified by the user
+
+The rest of operations are self-explanatory: `#insert`, `#update`,
+`#delete`, `#columnDelete`, `#rangeDelete`, `#sliceDelete`,
+`#partitionDelete`, and their plural counterparts.
+
+After history generated by `HistoryBuilder` is replayed using
+`ReplayingVisitor`, you can use any model (`QuiescentChecker` by
+default) to validate queries. Queries can be provided manually or
+generated using `QueryGenerator` or `TypedQueryGenerator`.
+
+# Basic Terminology
+
+ * Inflate / inflatable: a process of producing a value (for example, string, or a blob)
+ from a `long` descriptor that uniquely identifies the value.
+ See [data generation](https://github.com/apache/cassandra-harry#data-generation) section
+ of this guide for more details.
+ * Deflate / deflatable: a process of producing the descriptor the value was inflated
+ from during verification. See [model](https://github.com/apache/cassandra-harry#model)
+ section of this guide for more details.
+
+For definitions of logical timestamp, descriptor, and other entities used during
+inflation and deflation, refer to [formal relationships](https://github.com/apache/cassandra-harry#formal-relations-between-entities)
+section.
+
+# Features
+
+Currently, Harry can exercise the following Cassandra functionality:
+
+ * Supported data types: `int8`, `int16`, `int32`, `int64`, `boolean`, `float`,
+ `double`, `ascii`, `uuid`, `timestamp`. Collections are only _inflatable_.
+ * Random schema generation, with an arbitrary number of partition and clustering
+ keys.
+ * Schemas with arbitrary `CLUSTERING ORDER BY`
+ * Randomly generated `INSERT` and `UPDATE` queries with all columns or arbitrary
+ column subset
+ * Randomly generated `DELETE` queries: for a single column, single row, or
+ a range of rows
+ * Inflating and validating entire partitions (with allowed in-flight queries)
+ * Inflating and validating random `SELECT` queries: single row, slices (with single
+ open end), and ranges (with both ends of clusterings specified)
+
+Inflating partitions is done using [Reconciler](https://github.com/apache/cassandra-harry/blob/master/harry-core/src/harry/reconciler/Reconciler.java).
+Validating partitions and random queries can be done using [Quiescent Checker](https://github.com/apache/cassandra-harry/blob/master/harry-core/src/harry/model/QuiescentChecker.java)
+and [Exhaustive Checker](https://github.com/apache/cassandra-harry/blob/master/harry-core/src/harry/model/ExhaustiveChecker.java).
+
+## What's missing
+
+Harry is by no means feature-complete. Main things that are missing are:
+
+ * Some types (such as collections) are not deflatable
+ * Some types are implemented but are not hooked up (`blob` and `text`) to DSL/generator
+ * Partition deletions are not implemented
+ * 2i queries are not implemented
+ * Compact storage is not implemented
+ * Static columns are not implemented
+ * Fault injection is not implemented
+ * Runner and scheduler are rather rudimentary and require significant rework and proper scheduling
+ * TTL is not supported
+ * Some SELECT queries are not supported: `LIMIT`, `IN`, `GROUP BY`, token range queries
+ * Partition deletions are not implemented
+ * Pagination is not implemented
+
+Some things, even though are implemented, can be improved or optimized:
+
+ * RNG should be able to yield less than 64 bits of entropy per step
+ * State tracking should be done in a compact off-heap data stucture
+ * Inflated partition state and per-row operation log should be done in a compact
+ off-heap data structure
+ * Exhaustive checker can be significantly optimized
+ * Harry shouldn't rely on java-driver for query generation
+ * Exhaustive checker should use more precise information from data tracker, not
+ just watermarks
+ * Decision-making about _when_ we visit partitions and/or rows should be improved
+
+This list of improvements is incomplete, and should only give the reader a rough
+idea about the state of the project. Main goal for the initial release was to make it
+useful, now we can make it fast and feature-complete!
+
+# Goals
+
_Reproducibility_ is achieved by using the PCG family of random number
generators and generating schema, configuration, and every step of the workload
from the repeatable sequence of random numbers. Schema and configuration are
@@ -58,6 +220,60 @@ visited for a logical timestamp, how many operations there will be in batch,
what kind of operations there will and how often each kind of operation is going
to occur.
+# Implementing Tests
+
+All Harry components are pluggable and can be redefined. However, many
+of the default implementations will cover most of the use-cases, so in
+this guide we’ll focus on ones that are most often used to implement
+different use cases:
+
+ * System Under Test: defines how Harry can communicate with
+ Cassandra instances and issue common queries. Examples of a system
+ under test can be a CCM cluster, a “real” Cassandra cluster, or an
+ in-JVM dtest cluster.
+ * Visitor: defines behaviour that gets triggered at a specific
+ logical timestamp. One of the default implementations is
+ MutatingVisitor, which executes write workload against
+ SystemUnderTest. Examples of a visitor, besides a mutating visitor,
+ could be a validator that uses the model to validate results of
+ different queries, a repair runner, or a fault injector.
+ * Model: validates results of read queries by comparing its own
+ internal representation against the results returned by system
+ under test. You can find three simplified implementations of
+ model in this document: Visible Rows Checker, Quiescent Checker,
+ and an Exhaustive Checker.
+ * Runner: defines how operations defined by visitors are
+ executed. Harry includes two default implementations: a sequential
+ and a concurrent runner. Sequential runner allows no overlap
+ between different visitors or logical timestamps. Concurrent
+ runner allows visitors for different timestamps to overlap.
+
+System under test is the simplest one to implement: you only need a
+way to execute Cassandra queries. At the moment of writing, all custom
+things, such as nodetool commands, failure injection, etc, are
+implemented using a SUT / visitor combo: visitor knows about internals
+of the cluster it is dealing with.
+
+Generally, visitor has to follow the rules specified by
+DescriptorSelector and PdSelector: (it can only visit issue mutations
+against the partition that PdSelector has picked for this LTS), and
+DescriptorSelector (it can visit exactly
+DescriptorSelector#numberOfModifications rows within this partition,
+operations have to have a type specified by #operationKind, clustering
+and value descriptors have to be in accordance with
+DescriptorSelector#cd and DescriptorSelector#vds). The reason for
+these limitations is because model has to be able to reproduce the
+exact sequence of events that was applied to system under test.
+
+Default implementations of partition and clustering descriptors, used
+in fuzz mode allow to optimise verification. For example, it is
+possible to go find logical timestamps that are visiting the same
+partition as any given logical timestamp. When running Harry in
+unit-test mode, we use a special generating visitor that keeps an
+entire given sequence of events in memory rather than producing it on
+the fly. For reasons of efficiency, we do not use generating visitors
+for generating and verifying large datasets.
+
# Formal Relations Between Entities
To be able to implement efficient models, we had to reduce the amount of state
diff --git a/conf/default.yaml b/conf/default.yaml
index cdbc34c..c8d00b3 100644
--- a/conf/default.yaml
+++ b/conf/default.yaml
@@ -85,7 +85,7 @@ clustering_descriptor_selector:
# and model state.
runner:
sequential:
- partition_visitors:
+ visitors:
- logging:
row_visitor:
mutating: {}
diff --git a/conf/external.yaml b/conf/external.yaml
index 52943b8..a1432be 100644
--- a/conf/external.yaml
+++ b/conf/external.yaml
@@ -92,7 +92,7 @@ clustering_descriptor_selector:
# and model state.
runner:
sequential:
- partition_visitors:
+ visitors:
- logging:
row_visitor:
mutating: {}
diff --git a/harry-core/src/harry/core/Configuration.java b/harry-core/src/harry/core/Configuration.java
index 0e065d5..c6fecc7 100644
--- a/harry-core/src/harry/core/Configuration.java
+++ b/harry-core/src/harry/core/Configuration.java
@@ -47,16 +47,16 @@ import harry.model.clock.OffsetClock;
import harry.model.sut.PrintlnSut;
import harry.model.sut.SystemUnderTest;
import harry.visitors.AllPartitionsValidator;
-import harry.visitors.CorruptingPartitionVisitor;
+import harry.visitors.CorruptingVisitor;
import harry.runner.DataTracker;
import harry.runner.DefaultDataTracker;
-import harry.visitors.LoggingPartitionVisitor;
-import harry.visitors.MutatingPartitionVisitor;
+import harry.visitors.LoggingVisitor;
+import harry.visitors.MutatingVisitor;
import harry.visitors.MutatingRowVisitor;
-import harry.visitors.Operation;
-import harry.visitors.ParallelRecentPartitionValidator;
-import harry.visitors.PartitionVisitor;
-import harry.visitors.RecentPartitionValidator;
+import harry.visitors.OperationExecutor;
+import harry.visitors.ParallelRecentValidator;
+import harry.visitors.Visitor;
+import harry.visitors.RecentValidator;
import harry.runner.Runner;
import harry.visitors.Sampler;
import harry.util.BitSet;
@@ -87,12 +87,12 @@ public class Configuration
mapper.registerSubtypes(DefaultSchemaProviderConfiguration.class);
mapper.registerSubtypes(MutatingRowVisitorConfiguration.class);
- mapper.registerSubtypes(MutatingPartitionVisitorConfiguation.class);
- mapper.registerSubtypes(LoggingPartitionVisitorConfiguration.class);
+ mapper.registerSubtypes(MutatingVisitorConfiguation.class);
+ mapper.registerSubtypes(LoggingVisitorConfiguration.class);
mapper.registerSubtypes(AllPartitionsValidatorConfiguration.class);
- mapper.registerSubtypes(ParallelRecentPartitionValidator.ParallelRecentPartitionValidatorConfig.class);
+ mapper.registerSubtypes(ParallelRecentValidator.ParallelRecentValidatorConfig.class);
mapper.registerSubtypes(Sampler.SamplerConfiguration.class);
- mapper.registerSubtypes(CorruptingPartitionVisitorConfiguration.class);
+ mapper.registerSubtypes(CorruptingVisitorConfiguration.class);
mapper.registerSubtypes(RecentPartitionsValidatorConfiguration.class);
mapper.registerSubtypes(FixedSchemaProviderConfiguration.class);
mapper.registerSubtypes(AlwaysSamePartitionSelector.AlwaysSamePartitionSelectorConfiguration.class);
@@ -562,38 +562,38 @@ public class Configuration
public static class ConcurrentRunnerConfig implements RunnerConfiguration
{
public final int concurrency;
- public final List<PartitionVisitorConfiguration> partition_visitor_factories;
+ public final List<VisitorConfiguration> visitor_factories;
@JsonCreator
public ConcurrentRunnerConfig(@JsonProperty(value = "concurrency", defaultValue = "2") int concurrency,
- @JsonProperty(value = "partition_visitors") List<PartitionVisitorConfiguration> partitionVisitors)
+ @JsonProperty(value = "visitors") List<VisitorConfiguration> visitors)
{
this.concurrency = concurrency;
- this.partition_visitor_factories = partitionVisitors;
+ this.visitor_factories = visitors;
}
@Override
public Runner make(Run run, Configuration config)
{
- return new Runner.ConcurrentRunner(run, config, concurrency, partition_visitor_factories);
+ return new Runner.ConcurrentRunner(run, config, concurrency, visitor_factories);
}
}
@JsonTypeName("sequential")
public static class SequentialRunnerConfig implements RunnerConfiguration
{
- public final List<PartitionVisitorConfiguration> partition_visitor_factories;
+ public final List<VisitorConfiguration> visitor_factories;
@JsonCreator
- public SequentialRunnerConfig(@JsonProperty(value = "partition_visitors") List<PartitionVisitorConfiguration> partitionVisitors)
+ public SequentialRunnerConfig(@JsonProperty(value = "visitors") List<VisitorConfiguration> visitors)
{
- this.partition_visitor_factories = partitionVisitors;
+ this.visitor_factories = visitors;
}
@Override
public Runner make(Run run, Configuration config)
{
- return new Runner.SequentialRunner(run, config, partition_visitor_factories);
+ return new Runner.SequentialRunner(run, config, visitor_factories);
}
}
@@ -923,49 +923,49 @@ public class Configuration
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT)
- public interface PartitionVisitorConfiguration extends PartitionVisitor.PartitionVisitorFactory
+ public interface VisitorConfiguration extends Visitor.VisitorFactory
{
}
@JsonTypeName("mutating")
- public static class MutatingPartitionVisitorConfiguation implements PartitionVisitorConfiguration
+ public static class MutatingVisitorConfiguation implements VisitorConfiguration
{
public final RowVisitorConfiguration row_visitor;
@JsonCreator
- public MutatingPartitionVisitorConfiguation(@JsonProperty("row_visitor") RowVisitorConfiguration row_visitor)
+ public MutatingVisitorConfiguation(@JsonProperty("row_visitor") RowVisitorConfiguration row_visitor)
{
this.row_visitor = row_visitor;
}
@Override
- public PartitionVisitor make(Run run)
+ public Visitor make(Run run)
{
- return new MutatingPartitionVisitor(run, row_visitor);
+ return new MutatingVisitor(run, row_visitor);
}
}
@JsonTypeName("logging")
- public static class LoggingPartitionVisitorConfiguration implements PartitionVisitorConfiguration
+ public static class LoggingVisitorConfiguration implements VisitorConfiguration
{
protected final RowVisitorConfiguration row_visitor;
@JsonCreator
- public LoggingPartitionVisitorConfiguration(@JsonProperty("row_visitor") RowVisitorConfiguration row_visitor)
+ public LoggingVisitorConfiguration(@JsonProperty("row_visitor") RowVisitorConfiguration row_visitor)
{
this.row_visitor = row_visitor;
}
@Override
- public PartitionVisitor make(Run run)
+ public Visitor make(Run run)
{
- return new LoggingPartitionVisitor(run, row_visitor);
+ return new LoggingVisitor(run, row_visitor);
}
}
@JsonTypeName("validate_all_partitions")
- public static class AllPartitionsValidatorConfiguration implements Configuration.PartitionVisitorConfiguration
+ public static class AllPartitionsValidatorConfiguration implements VisitorConfiguration
{
public final int concurrency;
public final int trigger_after;
@@ -981,31 +981,31 @@ public class Configuration
this.modelConfiguration = model;
}
- public PartitionVisitor make(Run run)
+ public Visitor make(Run run)
{
return new AllPartitionsValidator(concurrency, trigger_after, run, modelConfiguration);
}
}
@JsonTypeName("corrupt")
- public static class CorruptingPartitionVisitorConfiguration implements Configuration.PartitionVisitorConfiguration
+ public static class CorruptingVisitorConfiguration implements VisitorConfiguration
{
public final int trigger_after;
@JsonCreator
- public CorruptingPartitionVisitorConfiguration(@JsonProperty("trigger_after") int trigger_after)
+ public CorruptingVisitorConfiguration(@JsonProperty("trigger_after") int trigger_after)
{
this.trigger_after = trigger_after;
}
- public PartitionVisitor make(Run run)
+ public Visitor make(Run run)
{
- return new CorruptingPartitionVisitor(trigger_after, run);
+ return new CorruptingVisitor(trigger_after, run);
}
}
@JsonTypeName("validate_recent_partitions")
- public static class RecentPartitionsValidatorConfiguration implements Configuration.PartitionVisitorConfiguration
+ public static class RecentPartitionsValidatorConfiguration implements VisitorConfiguration
{
public final int partition_count;
public final int trigger_after;
@@ -1026,14 +1026,14 @@ public class Configuration
}
@Override
- public PartitionVisitor make(Run run)
+ public Visitor make(Run run)
{
- return new RecentPartitionValidator(partition_count, queries, trigger_after, run, modelConfiguration);
+ return new RecentValidator(partition_count, queries, trigger_after, run, modelConfiguration);
}
}
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT)
- public interface RowVisitorConfiguration extends Operation.RowVisitorFactory
+ public interface RowVisitorConfiguration extends OperationExecutor.RowVisitorFactory
{
}
@@ -1041,7 +1041,7 @@ public class Configuration
public static class MutatingRowVisitorConfiguration implements RowVisitorConfiguration
{
@Override
- public Operation make(Run run)
+ public OperationExecutor make(Run run)
{
return new MutatingRowVisitor(run);
}
diff --git a/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java b/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java
index e05dffd..ace55b4 100644
--- a/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java
+++ b/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java
@@ -77,7 +77,7 @@ public class AddExtraRowCorruptor implements QueryResponseCorruptor
return false;
}
- long[] vds = descriptorSelector.vds(query.pd, cd, maxLts, 0, schema);
+ long[] vds = descriptorSelector.vds(query.pd, cd, maxLts, 0, OpSelectors.OperationKind.INSERT, schema);
// We do not know if the row was deleted. We could try inferring it, but that
// still won't help since we can't use it anyways, since collisions between a
diff --git a/harry-core/src/harry/dsl/HistoryBuilder.java b/harry-core/src/harry/dsl/HistoryBuilder.java
new file mode 100644
index 0000000..5e1ff6f
--- /dev/null
+++ b/harry-core/src/harry/dsl/HistoryBuilder.java
@@ -0,0 +1,625 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.dsl;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.function.Consumer;
+import java.util.function.LongSupplier;
+
+import harry.core.Run;
+import harry.model.OpSelectors;
+import harry.visitors.ReplayingVisitor;
+import harry.visitors.VisitExecutor;
+
+import static harry.model.OpSelectors.DefaultPdSelector.PARTITION_DESCRIPTOR_STREAM_ID;
+
+// TODO: we could use some sort of compact data structure or file format for navigable operation history
+public class HistoryBuilder implements Iterable<ReplayingVisitor.Visit>
+{
+ private final Run run;
+ private final List<ReplayingVisitor.Visit> log;
+
+ private long lts;
+ private final Set<Long> pds = new HashSet<>();
+
+ public Map<Long, NavigableSet<Long>> pdToLtsMap = new HashMap<>();
+
+ private int partitions;
+
+ public HistoryBuilder(Run run)
+ {
+ this.run = run;
+ this.log = new ArrayList<>();
+ this.lts = 0;
+ this.partitions = 0;
+
+ assert run.pdSelector instanceof PdSelector;
+ ((PdSelector) run.pdSelector).historyBuilder = this;
+ }
+
+ public Iterator<ReplayingVisitor.Visit> iterator()
+ {
+ return log.iterator();
+ }
+
+ public static class PdSelector extends OpSelectors.PdSelector
+ {
+ // We can only lazy-initialise it since history builder is created after pd selector
+ private HistoryBuilder historyBuilder;
+
+ protected long pd(long lts)
+ {
+ return historyBuilder.log.get((int) lts).pd;
+ }
+
+ public long nextLts(long lts)
+ {
+ Long next = historyBuilder.pdToLtsMap.get(pd(lts)).higher(lts);
+ if (null == next)
+ return -1;
+ return next;
+ }
+
+ public long prevLts(long lts)
+ {
+ Long prev = historyBuilder.pdToLtsMap.get(pd(lts)).lower(lts);
+ if (null == prev)
+ return -1;
+ return prev;
+ }
+
+ public long maxLtsFor(long pd)
+ {
+ return historyBuilder.pdToLtsMap.get(pd).last();
+ }
+
+ public long minLtsAt(long position)
+ {
+ return historyBuilder.pdToLtsMap.get(historyBuilder.pd(position)).first();
+ }
+
+ public long minLtsFor(long pd)
+ {
+ return historyBuilder.pdToLtsMap.get(pd).first();
+ }
+
+ public long positionFor(long lts)
+ {
+ return historyBuilder.position(pd(lts));
+ }
+ }
+
+ private static abstract class Step
+ {
+ public abstract ReplayingVisitor.Batch toBatch(long pd, long lts, long m, LongSupplier opIdSupplier);
+ }
+
+ private class BatchStep extends Step
+ {
+ private final List<OperationStep> steps;
+
+ protected BatchStep(List<OperationStep> steps)
+ {
+ this.steps = steps;
+ }
+
+ public ReplayingVisitor.Batch toBatch(long pd, long lts, long m, LongSupplier opIdSupplier)
+ {
+ ReplayingVisitor.Operation[] ops = new ReplayingVisitor.Operation[steps.size()];
+ for (int i = 0; i < ops.length; i++)
+ {
+ OperationStep opStep = steps.get(i);
+ long opId = opIdSupplier.getAsLong();
+ long cd = HistoryBuilder.this.cd(pd, lts, opId);
+ ops[i] = op(cd, opId, opStep.opType);
+ }
+
+ return HistoryBuilder.batch(m, ops);
+ }
+ }
+
+ private class OperationStep extends Step
+ {
+ private final OpSelectors.OperationKind opType;
+
+ protected OperationStep(OpSelectors.OperationKind opType)
+ {
+ this.opType = opType;
+ }
+
+ public ReplayingVisitor.Batch toBatch(long pd, long lts, long m, LongSupplier opIdSupplier)
+ {
+ long opId = opIdSupplier.getAsLong();
+ long cd = HistoryBuilder.this.cd(pd, lts, opId);
+ return HistoryBuilder.batch(m,
+ HistoryBuilder.op(cd, opIdSupplier.getAsLong(), opType));
+ }
+ }
+
+ public PartitionBuilder nextPartition()
+ {
+ long pd = pd(partitions++);
+ return new PartitionBuilder(pd);
+ }
+
+ // Ideally, we'd like to make these more generic
+ private long pd(long position)
+ {
+ long pd = run.schemaSpec.adjustPdEntropy(run.rng.prev(position, PARTITION_DESCRIPTOR_STREAM_ID));
+ pds.add(pd);
+ return pd;
+ }
+
+ private long position(long pd)
+ {
+ return run.rng.next(pd, PARTITION_DESCRIPTOR_STREAM_ID);
+ }
+
+ protected long cd(long pd, long lts, long opId)
+ {
+ return run.descriptorSelector.cd(pd, lts, opId, run.schemaSpec);
+ }
+
+ public class PartitionBuilder implements OperationBuilder<PartitionBuilder>
+ {
+
+ final List<Step> steps = new ArrayList<>();
+ final long pd;
+
+ boolean strictOrder = true;
+ boolean sequentially = true;
+
+ boolean finished = false;
+
+ public PartitionBuilder(long pd)
+ {
+ this.pd = pd;
+ }
+
+ public BatchBuilder<PartitionBuilder> batch()
+ {
+ return new BatchBuilder<>(this, steps::add);
+ }
+
+ /**
+ * Execute operations listed by users of this PartitionBuilder with same logical timestamp.
+ */
+ public PartitionBuilder simultaneously()
+ {
+ this.sequentially = false;
+ return this;
+ }
+
+ /**
+ * Execute operations listed by users of this PartitionBuilder with monotonically increasing timestamps,
+ * giving each operation its own timestamp. Timestamp order can be determined by `#randomOrder` / `#strictOrder`.
+ */
+ public PartitionBuilder sequentially()
+ {
+ this.sequentially = true;
+ return this;
+ }
+
+ /**
+ * Execute operations listed by users of this PartitionBuilder in random order
+ */
+ public PartitionBuilder randomOrder()
+ {
+ strictOrder = false;
+ return this;
+ }
+
+ /**
+ * Execute operations listed by users of this PartitionBuilder in the order given by the user
+ */
+ public PartitionBuilder strictOrder()
+ {
+ strictOrder = true;
+ return this;
+ }
+
+ public PartitionBuilder partitionDelete()
+ {
+ return step(OpSelectors.OperationKind.DELETE_PARTITION);
+ }
+
+ public PartitionBuilder partitionDeletions(int n)
+ {
+ for (int i = 0; i < n; i++)
+ partitionDelete();
+ return this;
+ }
+
+ public PartitionBuilder update()
+ {
+ return step(OpSelectors.OperationKind.UPDATE);
+ }
+
+ public PartitionBuilder updates(int n)
+ {
+ for (int i = 0; i < n; i++)
+ update();
+ return this;
+ }
+
+ public PartitionBuilder insert()
+ {
+ return step(OpSelectors.OperationKind.INSERT);
+ }
+
+ public PartitionBuilder inserts(int n)
+ {
+ for (int i = 0; i < n; i++)
+ insert();
+ return this;
+ }
+
+ public PartitionBuilder delete()
+ {
+ return step(OpSelectors.OperationKind.DELETE_ROW);
+ }
+
+ public PartitionBuilder deletes(int n)
+ {
+ for (int i = 0; i < n; i++)
+ delete();
+ return this;
+ }
+
+ public PartitionBuilder columnDelete()
+ {
+ return step(OpSelectors.OperationKind.DELETE_COLUMN_WITH_STATICS);
+ }
+
+ public PartitionBuilder columnDeletes(int n)
+ {
+ for (int i = 0; i < n; i++)
+ columnDelete();
+
+ return this;
+ }
+
+ public PartitionBuilder rangeDelete()
+ {
+ return step(OpSelectors.OperationKind.DELETE_RANGE);
+ }
+
+ public PartitionBuilder rangeDeletes(int n)
+ {
+ for (int i = 0; i < n; i++)
+ rangeDelete();
+
+ return this;
+ }
+
+ public PartitionBuilder sliceDelete()
+ {
+ return step(OpSelectors.OperationKind.DELETE_SLICE);
+ }
+
+ public PartitionBuilder sliceDeletes(int n)
+ {
+ for (int i = 0; i < n; i++)
+ sliceDelete();
+
+ return this;
+ }
+
+ public PartitionBuilder partitionBuilder()
+ {
+ return this;
+ }
+
+ public HistoryBuilder finish()
+ {
+ assert !finished;
+ finished = true;
+
+ if (!strictOrder)
+ // TODO: In the future/for large sets we could avoid generating the values and just generate them on the fly:
+ // https://lemire.me/blog/2017/09/18/visiting-all-values-in-an-array-exactly-once-in-random-order/
+ // we could just save the rules for generation, for example
+ Collections.shuffle(steps);
+
+ addSteps(steps);
+ steps.clear();
+ return HistoryBuilder.this;
+ }
+
+ void addSteps(List<Step> steps)
+ {
+ List<ReplayingVisitor.Batch> batches = new ArrayList<>();
+ Counter m = new Counter();
+ Counter opId = new Counter();
+ for (Step step : steps)
+ {
+ batches.add(step.toBatch(pd, lts, m.get(), opId::getAndIncrement));
+
+ if (sequentially)
+ {
+ assert lts == log.size();
+ addToLog(pd, batches);
+ m.reset();
+ }
+ else
+ {
+ m.increment();
+ }
+
+ opId.reset();
+ }
+
+ // If we were generating steps for the partition with same LTS, add remaining steps
+ if (!batches.isEmpty())
+ {
+ assert !sequentially;
+ addToLog(pd, batches);
+ }
+ }
+
+ PartitionBuilder step(OpSelectors.OperationKind opType)
+ {
+ steps.add(new OperationStep(opType));
+ return this;
+ }
+ }
+
+ private void addToLog(long pd, List<ReplayingVisitor.Batch> batches)
+ {
+ pdToLtsMap.compute(pd, (ignore, ltss) -> {
+ if (null == ltss)
+ ltss = new TreeSet<>();
+ ltss.add(lts);
+ return ltss;
+ });
+
+ log.add(visit(lts++, pd, batches.toArray(new ReplayingVisitor.Batch[0])));
+ batches.clear();
+ }
+
+ private static class Counter
+ {
+ long i;
+ void reset()
+ {
+ i = 0;
+ }
+
+ long increment()
+ {
+ return i++;
+ }
+
+ long getAndIncrement()
+ {
+ return i++;
+ }
+
+ long get()
+ {
+ return i;
+ }
+ }
+
+ public class BatchBuilder<T extends OperationBuilder<?>> implements OperationBuilder<BatchBuilder<T>>
+ {
+ final T operationBuilder;
+ final List<OperationStep> steps = new ArrayList<>();
+ final Consumer<Step> addStep;
+ boolean strictOrder;
+
+ boolean finished = false;
+ public BatchBuilder(T operationBuilder,
+ Consumer<Step> addStep)
+ {
+ this.operationBuilder = operationBuilder;
+ this.addStep = addStep;
+ }
+
+ public BatchBuilder<T> randomOrder()
+ {
+ this.strictOrder = false;
+ return this;
+ }
+
+ public BatchBuilder<T> strictOrder()
+ {
+ this.strictOrder = true;
+ return this;
+ }
+
+ public T finish()
+ {
+ assert !finished;
+ finished = true;
+ if (!strictOrder)
+ // TODO
+ Collections.shuffle(steps);
+
+ addStep.accept(new BatchStep(steps));
+ return operationBuilder;
+ }
+
+ public BatchBuilder<T> partitionDelete()
+ {
+ return step(OpSelectors.OperationKind.DELETE_PARTITION);
+ }
+
+ public BatchBuilder<T> partitionDeletions(int n)
+ {
+ for (int i = 0; i < n; i++)
+ partitionDelete();
+ return this;
+ }
+
+ public BatchBuilder<T> update()
+ {
+ return step(OpSelectors.OperationKind.UPDATE_WITH_STATICS);
+ }
+
+ public BatchBuilder<T> updates(int n)
+ {
+ for (int i = 0; i < n; i++)
+ update();
+ return this;
+ }
+
+ public BatchBuilder<T> insert()
+ {
+ return step(OpSelectors.OperationKind.INSERT_WITH_STATICS);
+ }
+
+ BatchBuilder<T> step(OpSelectors.OperationKind opType)
+ {
+ steps.add(new OperationStep(opType));
+ return this;
+ }
+
+ public BatchBuilder<T> inserts(int n)
+ {
+ for (int i = 0; i < n; i++)
+ insert();
+ return this;
+ }
+
+ public BatchBuilder<T> delete()
+ {
+ return step(OpSelectors.OperationKind.DELETE_ROW);
+ }
+
+ public BatchBuilder<T> deletes(int n)
+ {
+ for (int i = 0; i < n; i++)
+ delete();
+ return this;
+ }
+
+ public BatchBuilder<T> columnDelete()
+ {
+ return step(OpSelectors.OperationKind.DELETE_COLUMN_WITH_STATICS);
+ }
+
+ public BatchBuilder<T> columnDeletes(int n)
+ {
+ for (int i = 0; i < n; i++)
+ columnDelete();
+
+ return this;
+ }
+
+ public BatchBuilder<T> rangeDelete()
+ {
+ return step(OpSelectors.OperationKind.DELETE_RANGE);
+ }
+
+ public BatchBuilder<T> rangeDeletes(int n)
+ {
+ for (int i = 0; i < n; i++)
+ rangeDelete();
+ return this;
+ }
+
+ public BatchBuilder<T> sliceDelete()
+ {
+ return step(OpSelectors.OperationKind.DELETE_SLICE);
+ }
+
+ public BatchBuilder<T> sliceDeletes(int n)
+ {
+ for (int i = 0; i < n; i++)
+ sliceDelete();
+ return this;
+ }
+
+ public PartitionBuilder partitionBuilder()
+ {
+ return operationBuilder.partitionBuilder();
+ }
+ }
+
+ public interface OperationBuilder<T extends OperationBuilder<?>>
+ {
+ T randomOrder();
+ T strictOrder();
+ T partitionDelete();
+ T partitionDeletions(int n);
+ T update();
+ T updates(int n);
+ T insert();
+ T inserts(int n);
+ T delete();
+ T deletes(int n);
+ T columnDelete();
+ T columnDeletes(int n);
+ T rangeDelete();
+ T rangeDeletes(int n);
+ T sliceDelete();
+ T sliceDeletes(int n);
+ PartitionBuilder partitionBuilder();
+ }
+
+ public static ReplayingVisitor.Visit visit(long lts, long pd, ReplayingVisitor.Batch... ops)
+ {
+ return new ReplayingVisitor.Visit(lts, pd, ops);
+ }
+
+ public static ReplayingVisitor.Batch batch(long m, ReplayingVisitor.Operation... ops)
+ {
+ return new ReplayingVisitor.Batch(m, ops);
+ }
+
+ public static ReplayingVisitor.Operation op(long cd, long opId, OpSelectors.OperationKind opType)
+ {
+ return new ReplayingVisitor.Operation(cd, opId, opType);
+ }
+
+ public ReplayingVisitor visitor(VisitExecutor executor)
+ {
+ return new ReplayingVisitor(executor)
+ {
+ public Visit getVisit(long lts)
+ {
+ assert log.size() > lts : String.format("Log: %s, lts: %d", log, lts);
+ return log.get((int) lts);
+ }
+
+ public void replayAll(Run run)
+ {
+ long maxLts = HistoryBuilder.this.lts;
+ while (true)
+ {
+ long lts = run.clock.currentLts();
+ if (lts >= maxLts)
+ return;
+ visit(lts);
+ run.clock.nextLts();
+ }
+ }
+ };
+ }
+}
diff --git a/harry-core/src/harry/model/OpSelectors.java b/harry-core/src/harry/model/OpSelectors.java
index 3dd4a25..7e5fc17 100644
--- a/harry-core/src/harry/model/OpSelectors.java
+++ b/harry-core/src/harry/model/OpSelectors.java
@@ -82,6 +82,8 @@ public interface OpSelectors
long lts(long rts);
+ long currentLts();
+
long nextLts();
long maxLts();
@@ -122,6 +124,7 @@ public interface OpSelectors
public abstract long minLtsFor(long pd);
+ // TODO: right now, we can only calculate a position for 64-bit (in other words, full entropy) pds
public abstract long positionFor(long lts);
}
@@ -177,21 +180,22 @@ public interface OpSelectors
@VisibleForTesting
protected abstract long vd(long pd, long cd, long lts, long opId, int col);
- public long[] vds(long pd, long cd, long lts, long opId, SchemaSpec schema)
+ public long[] vds(long pd, long cd, long lts, long opId, OperationKind opType, SchemaSpec schema)
{
- BitSet setColumns = columnMask(pd, lts, opId);
+ BitSet setColumns = columnMask(pd, lts, opId, opType);
return descriptors(pd, cd, lts, opId, schema.regularColumns, schema.regularColumnsMask(), setColumns, schema.regularColumnsOffset);
}
- public long[] sds(long pd, long cd, long lts, long opId, SchemaSpec schema)
+ public long[] sds(long pd, long cd, long lts, long opId, OperationKind opType, SchemaSpec schema)
{
- BitSet setColumns = columnMask(pd, lts, opId);
+ BitSet setColumns = columnMask(pd, lts, opId, opType);
return descriptors(pd, cd, lts, opId, schema.staticColumns, schema.staticColumnsMask(), setColumns, schema.staticColumnsOffset);
}
private long[] descriptors(long pd, long cd, long lts, long opId, List<ColumnSpec<?>> columns, BitSet mask, BitSet setColumns, int offset)
{
- assert opId < opsPerModification(lts) * numberOfModifications(lts) : String.format("Operation id %d exceeds the maximum expected number of operations %d", opId, opsPerModification(lts) * numberOfModifications(lts));
+ assert opId < opsPerModification(lts) * numberOfModifications(lts) : String.format("Operation id %d exceeds the maximum expected number of operations %d (%d * %d)",
+ opId, opsPerModification(lts) * numberOfModifications(lts), opsPerModification(lts), numberOfModifications(lts));
long[] descriptors = new long[columns.size()];
for (int i = 0; i < descriptors.length; i++)
@@ -216,7 +220,7 @@ public interface OpSelectors
public abstract OperationKind operationType(long pd, long lts, long opId);
- public abstract BitSet columnMask(long pd, long lts, long opId);
+ public abstract BitSet columnMask(long pd, long lts, long opId, OperationKind opType);
// TODO: why is this one unused?
public abstract long rowId(long pd, long lts, long cd);
@@ -671,8 +675,7 @@ public interface OpSelectors
public OperationKind operationType(long pd, long lts, long opId)
{
- OperationKind kind = operationType(pd, lts, opId, partitionLevelOperationsMask(pd, lts));
- return kind;
+ return operationType(pd, lts, opId, partitionLevelOperationsMask(pd, lts));
}
// TODO: create this bitset once per lts
@@ -687,16 +690,16 @@ public interface OpSelectors
return BitSet.create(partitionLevelOpsMask, totalOps);
}
- public OperationKind operationType(long pd, long lts, long opId, BitSet partitionLevelOperationsMask)
+ private OperationKind operationType(long pd, long lts, long opId, BitSet partitionLevelOperationsMask)
{
long descriptor = rng.randomNumber(pd ^ lts ^ opId, BITSET_IDX_STREAM);
return operationSelector.inflate(descriptor, partitionLevelOperationsMask.isSet((int) opId));
}
- public BitSet columnMask(long pd, long lts, long opId)
+ public BitSet columnMask(long pd, long lts, long opId, OperationKind opType)
{
long descriptor = rng.randomNumber(pd ^ lts ^ opId, BITSET_IDX_STREAM);
- return columnSelector.columnMask(operationType(pd, lts, opId), descriptor);
+ return columnSelector.columnMask(opType, descriptor);
}
public long vd(long pd, long cd, long lts, long opId, int col)
diff --git a/harry-core/src/harry/model/QuiescentChecker.java b/harry-core/src/harry/model/QuiescentChecker.java
index d17d6f4..a630df2 100644
--- a/harry-core/src/harry/model/QuiescentChecker.java
+++ b/harry-core/src/harry/model/QuiescentChecker.java
@@ -45,10 +45,14 @@ public class QuiescentChecker implements Model
public QuiescentChecker(Run run)
{
+ this(run, new Reconciler(run));
+ }
+
+ public QuiescentChecker(Run run, Reconciler reconciler)
+ {
this.clock = run.clock;
this.sut = run.sut;
-
- this.reconciler = new Reconciler(run);
+ this.reconciler = reconciler;
this.tracker = run.tracker;
this.schemaSpec = run.schemaSpec;
}
@@ -63,8 +67,10 @@ public class QuiescentChecker implements Model
long maxCompeteLts = tracker.maxConsecutiveFinished();
long maxSeenLts = tracker.maxStarted();
- assert maxCompeteLts == maxSeenLts : "Runner hasn't settled down yet. " +
- "Quiescent model can't be reliably used in such cases.";
+ assert maxCompeteLts == maxSeenLts : String.format("Runner hasn't settled down yet. " +
+ "Quiescent model can't be reliably used in such cases. " +
+ "Max complete: %d. Max seen: %d",
+ maxCompeteLts, maxSeenLts);
List<ResultSetRow> actualRows = rowsSupplier.get();
Iterator<ResultSetRow> actual = actualRows.iterator();
diff --git a/harry-core/src/harry/model/clock/ApproximateMonotonicClock.java b/harry-core/src/harry/model/clock/ApproximateMonotonicClock.java
index 1a64500..186a618 100644
--- a/harry-core/src/harry/model/clock/ApproximateMonotonicClock.java
+++ b/harry-core/src/harry/model/clock/ApproximateMonotonicClock.java
@@ -151,6 +151,11 @@ public class ApproximateMonotonicClock implements OpSelectors.MonotonicClock
throw new IllegalStateException("No thread should have changed LTS during rebase. " + lts.get());
}
+ public long currentLts()
+ {
+ return lts.get();
+ }
+
public long nextLts()
{
long current = lts.get();
diff --git a/harry-core/src/harry/model/clock/OffsetClock.java b/harry-core/src/harry/model/clock/OffsetClock.java
index 8c25394..6040125 100644
--- a/harry-core/src/harry/model/clock/OffsetClock.java
+++ b/harry-core/src/harry/model/clock/OffsetClock.java
@@ -47,6 +47,11 @@ public class OffsetClock implements OpSelectors.MonotonicClock
return rts - base;
}
+ public long currentLts()
+ {
+ return lts.get();
+ }
+
public long nextLts()
{
return lts.getAndIncrement();
diff --git a/harry-core/src/harry/operations/WriteHelper.java b/harry-core/src/harry/operations/WriteHelper.java
index 084f931..1fdf591 100644
--- a/harry-core/src/harry/operations/WriteHelper.java
+++ b/harry-core/src/harry/operations/WriteHelper.java
@@ -65,7 +65,8 @@ public class WriteHelper
}
b.append(") USING TIMESTAMP ")
- .append(timestamp);
+ .append(timestamp)
+ .append(";");
return new CompiledStatement(b.toString(), adjustArraySize(bindings, bindingsCount));
}
diff --git a/harry-core/src/harry/reconciler/Reconciler.java b/harry-core/src/harry/reconciler/Reconciler.java
index da2daa7..275b7c7 100644
--- a/harry-core/src/harry/reconciler/Reconciler.java
+++ b/harry-core/src/harry/reconciler/Reconciler.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.TreeMap;
+import java.util.function.Function;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,12 +35,14 @@ import harry.core.Run;
import harry.ddl.ColumnSpec;
import harry.ddl.SchemaSpec;
import harry.model.OpSelectors;
-import harry.visitors.AbstractPartitionVisitor;
-import harry.visitors.PartitionVisitor;
+import harry.visitors.GeneratingVisitor;
+import harry.visitors.Visitor;
import harry.operations.Query;
import harry.operations.QueryGenerator;
import harry.util.BitSet;
import harry.util.Ranges;
+import harry.visitors.ReplayingVisitor;
+import harry.visitors.VisitExecutor;
import static harry.generators.DataGenerators.NIL_DESCR;
import static harry.generators.DataGenerators.UNSET_DESCR;
@@ -64,12 +67,22 @@ public class Reconciler
private final QueryGenerator rangeSelector;
private final SchemaSpec schema;
+ private final Function<VisitExecutor, Visitor> visitorFactory;
+
public Reconciler(Run run)
{
+ this(run,
+ (processor) -> new GeneratingVisitor(run, processor));
+ }
+
+ public Reconciler(Run run,
+ Function<VisitExecutor, Visitor> visitorFactory)
+ {
this.descriptorSelector = run.descriptorSelector;
this.pdSelector = run.pdSelector;
this.schema = run.schemaSpec;
this.rangeSelector = run.rangeSelector;
+ this.visitorFactory = visitorFactory;
}
private final long debugCd = Long.getLong("harry.reconciler.debug_cd", -1L);
@@ -78,26 +91,20 @@ public class Reconciler
{
PartitionState partitionState = new PartitionState();
- class Processor extends AbstractPartitionVisitor
+ class Processor implements VisitExecutor
{
- public Processor(OpSelectors.PdSelector pdSelector, OpSelectors.DescriptorSelector descriptorSelector, SchemaSpec schema)
- {
- super(pdSelector, descriptorSelector, schema);
- }
-
// Whether or not a partition deletion was encountered on this LTS.
private boolean hadPartitionDeletion = false;
private final List<Ranges.Range> rangeDeletes = new ArrayList<>();
- private final List<Long> writes = new ArrayList<>();
- private final List<Long> columnDeletes = new ArrayList<>();
+ private final List<ReplayingVisitor.Operation> writes = new ArrayList<>();
+ private final List<ReplayingVisitor.Operation> columnDeletes = new ArrayList<>();
@Override
- public void operation(long lts, long pd, long cd, long m, long opId)
+ public void operation(long lts, long pd, long cd, long m, long opId, OpSelectors.OperationKind opType)
{
if (hadPartitionDeletion)
return;
- OpSelectors.OperationKind opType = descriptorSelector.operationType(pd, lts, opId);
switch (opType)
{
case DELETE_RANGE:
@@ -131,11 +138,12 @@ public class Reconciler
case UPDATE_WITH_STATICS:
if (debugCd != -1 && cd == debugCd)
logger.info("Writing {} ({}) at {}/{}", cd, opType, lts, opId);
- writes.add(opId);
+ // TODO: switch to Operation as an entity that can just be passed here
+ writes.add(new ReplayingVisitor.Operation(cd, opId, opType));
break;
case DELETE_COLUMN_WITH_STATICS:
case DELETE_COLUMN:
- columnDeletes.add(opId);
+ columnDeletes.add(new ReplayingVisitor.Operation(cd, opId, opType));
break;
default:
throw new IllegalStateException();
@@ -143,7 +151,7 @@ public class Reconciler
}
@Override
- protected void beforeLts(long lts, long pd)
+ public void beforeLts(long lts, long pd)
{
rangeDeletes.clear();
writes.clear();
@@ -152,25 +160,23 @@ public class Reconciler
}
@Override
- protected void afterLts(long lts, long pd)
+ public void afterLts(long lts, long pd)
{
if (hadPartitionDeletion)
return;
- outer: for (Long opIdBoxed : writes)
+ outer: for (ReplayingVisitor.Operation op : writes)
{
- long opId = opIdBoxed;
- long cd = descriptorSelector.cd(pd, lts, opId, schema);
+ long opId = op.opId;
+ long cd = op.cd;
- OpSelectors.OperationKind opType = descriptorSelector.operationType(pd, lts, opId);
-
- switch (opType)
+ switch (op.opType)
{
case INSERT_WITH_STATICS:
case UPDATE_WITH_STATICS:
// We could apply static columns during the first iteration, but it's more convenient
// to reconcile static-level deletions.
- partitionState.writeStaticRow(descriptorSelector.sds(pd, cd, lts, opId, schema),
+ partitionState.writeStaticRow(descriptorSelector.sds(pd, cd, lts, opId, op.opType, schema),
lts);
case INSERT:
case UPDATE:
@@ -192,28 +198,26 @@ public class Reconciler
}
partitionState.write(cd,
- descriptorSelector.vds(pd, cd, lts, opId, schema),
+ descriptorSelector.vds(pd, cd, lts, opId, op.opType, schema),
lts,
- opType == OpSelectors.OperationKind.INSERT || opType == OpSelectors.OperationKind.INSERT_WITH_STATICS);
+ op.opType == OpSelectors.OperationKind.INSERT || op.opType == OpSelectors.OperationKind.INSERT_WITH_STATICS);
break;
default:
- throw new IllegalStateException();
+ throw new IllegalStateException(op.opType.toString());
}
}
- outer: for (Long opIdBoxed : columnDeletes)
+ outer: for (ReplayingVisitor.Operation op : columnDeletes)
{
- long opId = opIdBoxed;
- long cd = descriptorSelector.cd(pd, lts, opId, schema);
-
- OpSelectors.OperationKind opType = descriptorSelector.operationType(pd, lts, opId);
+ long opId = op.opId;
+ long cd = op.cd;
- switch (opType)
+ switch (op.opType)
{
case DELETE_COLUMN_WITH_STATICS:
partitionState.deleteStaticColumns(lts,
schema.staticColumnsOffset,
- descriptorSelector.columnMask(pd, lts, opId),
+ descriptorSelector.columnMask(pd, lts, opId, op.opType),
schema.staticColumnsMask());
case DELETE_COLUMN:
if (!query.match(cd))
@@ -236,21 +240,27 @@ public class Reconciler
partitionState.deleteRegularColumns(lts,
cd,
schema.regularColumnsOffset,
- descriptorSelector.columnMask(pd, lts, opId),
+ descriptorSelector.columnMask(pd, lts, opId, op.opType),
schema.regularColumnsMask());
break;
}
}
}
+
+ @Override
+ public void afterBatch(long lts, long pd, long m) {}
+
+ @Override
+ public void beforeBatch(long lts, long pd, long m) {}
}
- PartitionVisitor partitionVisitor = new Processor(pdSelector, descriptorSelector, schema);
+ Visitor visitor = visitorFactory.apply(new Processor());
long currentLts = pdSelector.minLtsFor(pd);
while (currentLts <= maxLts && currentLts >= 0)
{
- partitionVisitor.visitPartition(currentLts);
+ visitor.visit(currentLts);
currentLts = pdSelector.nextLts(currentLts);
}
@@ -283,9 +293,9 @@ public class Reconciler
private void write(long cd,
long[] vds,
long lts,
- boolean writeParimaryKeyLiveness)
+ boolean writePrimaryKeyLiveness)
{
- rows.compute(cd, (cd_, current) -> updateRowState(current, schema.regularColumns, cd, vds, lts, writeParimaryKeyLiveness));
+ rows.compute(cd, (cd_, current) -> updateRowState(current, schema.regularColumns, cd, vds, lts, writePrimaryKeyLiveness));
}
private void delete(Ranges.Range range,
diff --git a/harry-core/src/harry/runner/DataTracker.java b/harry-core/src/harry/runner/DataTracker.java
index dab6fc9..9f518d1 100644
--- a/harry-core/src/harry/runner/DataTracker.java
+++ b/harry-core/src/harry/runner/DataTracker.java
@@ -18,8 +18,6 @@
package harry.runner;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonTypeName;
import harry.core.Configuration;
public interface DataTracker
diff --git a/harry-core/src/harry/runner/DefaultDataTracker.java b/harry-core/src/harry/runner/DefaultDataTracker.java
index 7b1412a..d5cdb4b 100644
--- a/harry-core/src/harry/runner/DefaultDataTracker.java
+++ b/harry-core/src/harry/runner/DefaultDataTracker.java
@@ -63,7 +63,10 @@ public class DefaultDataTracker implements DataTracker
private void recordEvent(long lts, boolean finished)
{
// all seen LTS are allowed to be "in-flight"
- maxSeenLts.getAndUpdate((old) -> Math.max(lts, old));
+ maxSeenLts.getAndUpdate((old) -> {
+ assert finished || lts > old : String.format("Attempting to reuse lts: %d. Max seen: %d", lts, old);
+ return Math.max(lts, old);
+ });
if (!finished)
return;
diff --git a/harry-core/src/harry/runner/Runner.java b/harry-core/src/harry/runner/Runner.java
index b172723..3a84d3e 100644
--- a/harry-core/src/harry/runner/Runner.java
+++ b/harry-core/src/harry/runner/Runner.java
@@ -39,7 +39,7 @@ import org.slf4j.LoggerFactory;
import harry.core.Configuration;
import harry.core.Run;
import harry.model.OpSelectors;
-import harry.visitors.PartitionVisitor;
+import harry.visitors.Visitor;
public abstract class Runner
@@ -140,21 +140,21 @@ public abstract class Runner
{
private final ScheduledExecutorService executor;
private final ScheduledExecutorService shutdownExceutor;
- private final List<PartitionVisitor> partitionVisitors;
+ private final List<Visitor> visitors;
private final Configuration config;
public SequentialRunner(Run run,
Configuration config,
- List<? extends PartitionVisitor.PartitionVisitorFactory> partitionVisitorFactories)
+ List<? extends Visitor.VisitorFactory> visitorFactories)
{
super(run, config);
this.executor = Executors.newSingleThreadScheduledExecutor();
this.shutdownExceutor = Executors.newSingleThreadScheduledExecutor();
this.config = config;
- this.partitionVisitors = new ArrayList<>();
- for (PartitionVisitor.PartitionVisitorFactory factory : partitionVisitorFactories)
- partitionVisitors.add(factory.make(run));
+ this.visitors = new ArrayList<>();
+ for (Visitor.VisitorFactory factory : visitorFactories)
+ visitors.add(factory.make(run));
}
public CompletableFuture<?> initAndStartAll()
@@ -173,7 +173,7 @@ public abstract class Runner
executor.submit(reportThrowable(() -> {
try
{
- SequentialRunner.run(partitionVisitors, run.clock, future,
+ SequentialRunner.run(visitors, run.clock, future,
() -> Thread.currentThread().isInterrupted() || future.isDone() || completed.get());
}
catch (Throwable t)
@@ -186,7 +186,7 @@ public abstract class Runner
return future;
}
- static void run(List<PartitionVisitor> visitors,
+ static void run(List<Visitor> visitors,
OpSelectors.MonotonicClock clock,
CompletableFuture<?> future,
BooleanSupplier exitCondition)
@@ -202,8 +202,8 @@ public abstract class Runner
{
try
{
- PartitionVisitor partitionVisitor = visitors.get(i);
- partitionVisitor.visitPartition(lts);
+ Visitor visitor = visitors.get(i);
+ visitor.visit(lts);
}
catch (Throwable t)
{
@@ -238,8 +238,8 @@ public abstract class Runner
{
private final ScheduledExecutorService executor;
private final ScheduledExecutorService shutdownExecutor;
- private final List<? extends PartitionVisitor.PartitionVisitorFactory> partitionVisitorFactories;
- private final List<PartitionVisitor> allVisitors;
+ private final List<? extends Visitor.VisitorFactory> visitorFactories;
+ private final List<Visitor> allVisitors;
private final int concurrency;
private final long runTime;
@@ -248,7 +248,7 @@ public abstract class Runner
public ConcurrentRunner(Run run,
Configuration config,
int concurrency,
- List<? extends PartitionVisitor.PartitionVisitorFactory> partitionVisitorFactories)
+ List<? extends Visitor.VisitorFactory> visitorFactories)
{
super(run, config);
this.concurrency = concurrency;
@@ -257,7 +257,7 @@ public abstract class Runner
// TODO: configure concurrency
this.executor = Executors.newScheduledThreadPool(concurrency);
this.shutdownExecutor = Executors.newSingleThreadScheduledExecutor();
- this.partitionVisitorFactories = partitionVisitorFactories;
+ this.visitorFactories = visitorFactories;
this.allVisitors = new CopyOnWriteArrayList<>();
}
@@ -276,13 +276,13 @@ public abstract class Runner
BooleanSupplier exitCondition = () -> Thread.currentThread().isInterrupted() || future.isDone();
for (int i = 0; i < concurrency; i++)
{
- List<PartitionVisitor> partitionVisitors = new ArrayList<>();
+ List<Visitor> visitors = new ArrayList<>();
executor.submit(reportThrowable(() -> {
- for (PartitionVisitor.PartitionVisitorFactory factory : partitionVisitorFactories)
- partitionVisitors.add(factory.make(run));
+ for (Visitor.VisitorFactory factory : visitorFactories)
+ visitors.add(factory.make(run));
- allVisitors.addAll(partitionVisitors);
- run(partitionVisitors, run.clock, exitCondition);
+ allVisitors.addAll(visitors);
+ run(visitors, run.clock, exitCondition);
},
future));
@@ -291,22 +291,22 @@ public abstract class Runner
return future;
}
- void run(List<PartitionVisitor> visitors,
+ void run(List<Visitor> visitors,
OpSelectors.MonotonicClock clock,
BooleanSupplier exitCondition)
{
while (!exitCondition.getAsBoolean())
{
long lts = clock.nextLts();
- for (PartitionVisitor visitor : visitors)
- visitor.visitPartition(lts);
+ for (Visitor visitor : visitors)
+ visitor.visit(lts);
}
}
public void shutdown() throws InterruptedException
{
logger.info("Shutting down...");
- for (PartitionVisitor visitor : allVisitors)
+ for (Visitor visitor : allVisitors)
visitor.shutdown();
shutdownExecutor.shutdownNow();
diff --git a/harry-core/src/harry/util/TestRunner.java b/harry-core/src/harry/util/TestRunner.java
index 4349fd2..cd0fd0e 100644
--- a/harry-core/src/harry/util/TestRunner.java
+++ b/harry-core/src/harry/util/TestRunner.java
@@ -19,8 +19,10 @@
package harry.util;
import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
+import java.util.function.Supplier;
import harry.generators.Generator;
import harry.generators.RandomGenerator;
@@ -59,9 +61,46 @@ public class TestRunner
}
}
+ public static <VISIT, MODEL, SUT> void test(Generator<VISIT> visitGenerator,
+ Supplier<MODEL> initializeModel,
+ Supplier<SUT> initializeSUT,
+ BiFunction<MODEL, VISIT, MODEL> applyToModel,
+ BiFunction<SUT, VISIT, SUT> applyToSut,
+ ThrowingBiConsumer<MODEL, SUT> afterAll) throws Throwable
+ {
+ MODEL model = initializeModel.get();
+ SUT sut = initializeSUT.get();
+ for (int i = 0; i < CYCLES; i++)
+ {
+ VISIT v = visitGenerator.generate(rand);
+ model = applyToModel.apply(model, v);
+ sut = applyToSut.apply(sut, v);
+ }
+ afterAll.accept(model, sut);
+ }
+
+ public static <VISIT, SUT> void test(Generator<VISIT> visitGenerator,
+ Supplier<SUT> initializeSUT,
+ BiFunction<SUT, VISIT, SUT> applyToSut,
+ Consumer<SUT> afterAll) throws Throwable
+ {
+ SUT sut = initializeSUT.get();
+ for (int i = 0; i < CYCLES; i++)
+ {
+ VISIT v = visitGenerator.generate(rand);
+ sut = applyToSut.apply(sut, v);
+ }
+ afterAll.accept(sut);
+ }
+
public static interface ThrowingConsumer<T>
{
void accept(T t) throws Throwable;
}
+
+ public static interface ThrowingBiConsumer<T1, T2>
+ {
+ void accept(T1 t1, T2 t2) throws Throwable;
+ }
}
diff --git a/harry-core/src/harry/visitors/AllPartitionsValidator.java b/harry-core/src/harry/visitors/AllPartitionsValidator.java
index 5a5a1c7..1b67a8e 100644
--- a/harry-core/src/harry/visitors/AllPartitionsValidator.java
+++ b/harry-core/src/harry/visitors/AllPartitionsValidator.java
@@ -38,7 +38,7 @@ import harry.operations.Query;
// This might be something that potentially grows into the validator described in the design doc;
// right now it's just a helper/container class
-public class AllPartitionsValidator implements PartitionVisitor
+public class AllPartitionsValidator implements Visitor
{
private static final Logger logger = LoggerFactory.getLogger(AllPartitionsValidator.class);
@@ -111,7 +111,7 @@ public class AllPartitionsValidator implements PartitionVisitor
private final AtomicLong maxPos = new AtomicLong(-1);
- public void visitPartition(long lts)
+ public void visit(long lts)
{
maxPos.updateAndGet(current -> Math.max(pdSelector.positionFor(lts), current));
diff --git a/harry-core/src/harry/visitors/CorruptingPartitionVisitor.java b/harry-core/src/harry/visitors/CorruptingVisitor.java
similarity index 94%
rename from harry-core/src/harry/visitors/CorruptingPartitionVisitor.java
rename to harry-core/src/harry/visitors/CorruptingVisitor.java
index 71c425b..26febeb 100644
--- a/harry-core/src/harry/visitors/CorruptingPartitionVisitor.java
+++ b/harry-core/src/harry/visitors/CorruptingVisitor.java
@@ -32,7 +32,7 @@ import harry.corruptor.QueryResponseCorruptor;
import harry.runner.HarryRunner;
import harry.operations.Query;
-public class CorruptingPartitionVisitor implements PartitionVisitor
+public class CorruptingVisitor implements Visitor
{
public static final Logger logger = LoggerFactory.getLogger(HarryRunner.class);
@@ -40,8 +40,8 @@ public class CorruptingPartitionVisitor implements PartitionVisitor
private final QueryResponseCorruptor[] corruptors;
private final int triggerAfter;
- public CorruptingPartitionVisitor(int triggerAfter,
- Run run)
+ public CorruptingVisitor(int triggerAfter,
+ Run run)
{
this.run = run;
this.triggerAfter = triggerAfter;
@@ -64,7 +64,7 @@ public class CorruptingPartitionVisitor implements PartitionVisitor
private final AtomicLong maxPos = new AtomicLong(-1);
- public void visitPartition(long lts)
+ public void visit(long lts)
{
maxPos.updateAndGet(current -> Math.max(run.pdSelector.positionFor(lts), current));
diff --git a/harry-core/src/harry/visitors/DelegatingVisitor.java b/harry-core/src/harry/visitors/DelegatingVisitor.java
new file mode 100644
index 0000000..f4994b1
--- /dev/null
+++ b/harry-core/src/harry/visitors/DelegatingVisitor.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.visitors;
+
+import harry.model.OpSelectors;
+
+public abstract class DelegatingVisitor implements Visitor, VisitExecutor
+{
+ protected VisitExecutor delegate;
+
+ public DelegatingVisitor(VisitExecutor delegate)
+ {
+ this.delegate = delegate;
+ }
+
+ public void beforeLts(long lts, long pd)
+ {
+ delegate.beforeLts(lts, pd);
+ }
+
+ public void afterLts(long lts, long pd)
+ {
+ delegate.afterLts(lts, pd);
+ }
+
+ public void beforeBatch(long lts, long pd, long m)
+ {
+ delegate.beforeBatch(lts, pd, m);
+ }
+
+ public void operation(long lts, long pd, long cd, long m, long opId, OpSelectors.OperationKind opType)
+ {
+ delegate.operation(lts, pd, cd, m, opId, opType);
+ }
+
+ public void afterBatch(long lts, long pd, long m)
+ {
+ delegate.afterBatch(lts, pd, m);
+ }
+
+ public void shutdown() throws InterruptedException
+ {
+ delegate.shutdown();
+ }
+}
diff --git a/harry-core/src/harry/visitors/AbstractPartitionVisitor.java b/harry-core/src/harry/visitors/GeneratingVisitor.java
similarity index 52%
rename from harry-core/src/harry/visitors/AbstractPartitionVisitor.java
rename to harry-core/src/harry/visitors/GeneratingVisitor.java
index 3f58e3a..78e2b07 100644
--- a/harry-core/src/harry/visitors/AbstractPartitionVisitor.java
+++ b/harry-core/src/harry/visitors/GeneratingVisitor.java
@@ -18,35 +18,32 @@
package harry.visitors;
+import harry.core.Run;
import harry.ddl.SchemaSpec;
import harry.model.OpSelectors;
-public abstract class AbstractPartitionVisitor implements PartitionVisitor
+public class GeneratingVisitor extends DelegatingVisitor
{
- protected final OpSelectors.PdSelector pdSelector;
- protected final OpSelectors.DescriptorSelector descriptorSelector;
- protected final SchemaSpec schema;
+ private final OpSelectors.PdSelector pdSelector;
+ private final OpSelectors.DescriptorSelector descriptorSelector;
+ private final SchemaSpec schema;
- public AbstractPartitionVisitor(AbstractPartitionVisitor visitor)
+ public GeneratingVisitor(Run run,
+ VisitExecutor delegate)
{
- this(visitor.pdSelector, visitor.descriptorSelector, visitor.schema);
+ super(delegate);
+ this.pdSelector = run.pdSelector;
+ this.descriptorSelector = run.descriptorSelector;
+ this.schema = run.schemaSpec;
}
- public AbstractPartitionVisitor(OpSelectors.PdSelector pdSelector,
- OpSelectors.DescriptorSelector descriptorSelector,
- SchemaSpec schema)
+ @Override
+ public void visit(long lts)
{
- this.pdSelector = pdSelector;
- this.descriptorSelector = descriptorSelector;
- this.schema = schema;
+ generate(lts, pdSelector.pd(lts, schema));
}
- public void visitPartition(long lts)
- {
- visitPartition(lts, pdSelector.pd(lts, schema));
- }
-
- private void visitPartition(long lts, long pd)
+ private void generate(long lts, long pd)
{
beforeLts(lts, pd);
@@ -60,36 +57,12 @@ public abstract class AbstractPartitionVisitor implements PartitionVisitor
{
long opId = m * opsPerModification + i;
long cd = descriptorSelector.cd(pd, lts, opId, schema);
- operation(lts, pd, cd, m, opId);
+ OpSelectors.OperationKind opType = descriptorSelector.operationType(pd, lts, opId);
+ operation(lts, pd, cd, m, opId, opType);
}
afterBatch(lts, pd, m);
}
afterLts(lts, pd);
}
-
- protected void beforeLts(long lts, long pd)
- {
- }
-
- protected void afterLts(long lts, long pd)
- {
- }
-
- protected void beforeBatch(long lts, long pd, long m)
- {
- }
-
- protected void operation(long lts, long pd, long cd, long m, long opId)
- {
-
- }
-
- protected void afterBatch(long lts, long pd, long m)
- {
- }
-
- public void shutdown() throws InterruptedException
- {
- }
-}
\ No newline at end of file
+}
diff --git a/harry-core/src/harry/visitors/LoggingPartitionVisitor.java b/harry-core/src/harry/visitors/LoggingPartitionVisitor.java
deleted file mode 100644
index 3e97ba3..0000000
--- a/harry-core/src/harry/visitors/LoggingPartitionVisitor.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package harry.visitors;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-
-import harry.core.Run;
-import harry.operations.CompiledStatement;
-
-public class LoggingPartitionVisitor extends MutatingPartitionVisitor
-{
- private final BufferedWriter operationLog;
-
- public LoggingPartitionVisitor(Run run, Operation.RowVisitorFactory rowVisitorFactory)
- {
- super(run, rowVisitorFactory);
-
- File f = new File("operation.log");
- try
- {
- operationLog = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(f)));
- }
- catch (FileNotFoundException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- public void afterLts(long lts, long pd)
- {
- super.afterLts(lts, pd);
- log("LTS: %d. Pd %d. Finished\n", lts, pd);
- }
-
- @Override
- protected CompiledStatement operationInternal(long lts, long pd, long cd, long m, long opId)
- {
- CompiledStatement statement = super.operationInternal(lts, pd, cd, m, opId);
-
- log(String.format("LTS: %d. Pd %d. Cd %d. M %d. OpId: %d Statement %s\n",
- lts, pd, cd, m, opId, statement));
-
- return statement;
- }
-
- private void log(String format, Object... objects)
- {
- try
- {
- operationLog.write(String.format(format, objects));
- operationLog.flush();
- }
- catch (IOException e)
- {
- // ignore
- }
- }
-}
diff --git a/harry-core/src/harry/visitors/LoggingVisitor.java b/harry-core/src/harry/visitors/LoggingVisitor.java
new file mode 100644
index 0000000..8deebde
--- /dev/null
+++ b/harry-core/src/harry/visitors/LoggingVisitor.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.visitors;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+
+import harry.core.Run;
+import harry.model.OpSelectors;
+import harry.operations.CompiledStatement;
+
+public class LoggingVisitor extends GeneratingVisitor
+{
+
+ public LoggingVisitor(Run run,
+ OperationExecutor.RowVisitorFactory rowVisitorFactory)
+ {
+ super(run, new LoggingVisitorExecutor(run, rowVisitorFactory.make(run)));
+ }
+
+ public static class LoggingVisitorExecutor extends MutatingVisitor.MutatingVisitExecutor
+ {
+ private final BufferedWriter operationLog;
+
+ public LoggingVisitorExecutor(Run run, OperationExecutor rowVisitor)
+ {
+ super(run, rowVisitor);
+
+ File f = new File("operation.log");
+ try
+ {
+ operationLog = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(f)));
+ }
+ catch (FileNotFoundException e)
+ {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public void afterLts(long lts, long pd)
+ {
+ super.afterLts(lts, pd);
+ log("LTS: %d. Pd %d. Finished\n", lts, pd);
+ }
+
+ @Override
+ protected CompiledStatement operationInternal(long lts, long pd, long cd, long m, long opId, OpSelectors.OperationKind opType)
+ {
+ CompiledStatement statement = super.operationInternal(lts, pd, cd, m, opId, opType);
+
+ log(String.format("LTS: %d. Pd %d. Cd %d. M %d. OpId: %d Statement %s\n",
+ lts, pd, cd, m, opId, statement));
+
+ return statement;
+ }
+
+ private void log(String format, Object... objects)
+ {
+ try
+ {
+ operationLog.write(String.format(format, objects));
+ operationLog.flush();
+ }
+ catch (IOException e)
+ {
+ // ignore
+ }
+ }
+ }
+}
diff --git a/harry-core/src/harry/visitors/MutatingPartitionVisitor.java b/harry-core/src/harry/visitors/MutatingPartitionVisitor.java
deleted file mode 100644
index 88456fe..0000000
--- a/harry-core/src/harry/visitors/MutatingPartitionVisitor.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package harry.visitors;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import harry.core.Run;
-import harry.model.OpSelectors;
-import harry.model.sut.SystemUnderTest;
-import harry.operations.CompiledStatement;
-import harry.runner.DataTracker;
-
-public class MutatingPartitionVisitor extends AbstractPartitionVisitor
-{
- private static final Logger logger = LoggerFactory.getLogger(MutatingPartitionVisitor.class);
-
- private final List<String> statements = new ArrayList<>();
- private final List<Object> bindings = new ArrayList<>();
-
- private final List<CompletableFuture<?>> futures = new ArrayList<>();
-
- protected final ScheduledExecutorService executor = Executors.newScheduledThreadPool(2);
- protected final DataTracker tracker;
- protected final SystemUnderTest sut;
- protected final Operation rowVisitor;
-
- public MutatingPartitionVisitor(Run run, Operation.RowVisitorFactory rowVisitorFactory)
- {
- super(run.pdSelector, run.descriptorSelector, run.schemaSpec);
- this.tracker = run.tracker;
- this.sut = run.sut;
- this.rowVisitor = rowVisitorFactory.make(run);
- }
-
- public void beforeLts(long lts, long pd)
- {
- tracker.started(lts);
- }
-
- public void afterLts(long lts, long pd)
- {
- for (CompletableFuture<?> future : futures)
- {
- try
- {
- future.get();
- }
- catch (Throwable t)
- {
- throw new IllegalStateException("Couldn't repeat operations within timeout bounds.", t);
- }
- }
- futures.clear();
- tracker.finished(lts);
- }
-
- public void beforeBatch(long lts, long pd, long m)
- {
- statements.clear();
- bindings.clear();
- }
-
- protected void operation(long lts, long pd, long cd, long m, long opId)
- {
- CompiledStatement statement = operationInternal(lts, pd, cd, m, opId);
- statements.add(statement.cql());
- for (Object binding : statement.bindings())
- bindings.add(binding);
- }
-
- protected CompiledStatement operationInternal(long lts, long pd, long cd, long m, long opId)
- {
- OpSelectors.OperationKind op = descriptorSelector.operationType(pd, lts, opId);
- return rowVisitor.perform(op, lts, pd, cd, opId);
- }
-
- public void afterBatch(long lts, long pd, long m)
- {
- if (statements.isEmpty())
- {
- logger.warn("Encountered an empty batch on {}", lts);
- return;
- }
-
- String query = String.join(" ", statements);
-
- if (statements.size() > 1)
- query = String.format("BEGIN UNLOGGED BATCH\n%s\nAPPLY BATCH;", query);
-
- Object[] bindingsArray = new Object[bindings.size()];
- bindings.toArray(bindingsArray);
-
- CompletableFuture<Object[][]> future = new CompletableFuture<>();
- executeAsyncWithRetries(future, new CompiledStatement(query, bindingsArray));
- futures.add(future);
-
- statements.clear();
- bindings.clear();
- }
-
- void executeAsyncWithRetries(CompletableFuture<Object[][]> future, CompiledStatement statement)
- {
- if (sut.isShutdown())
- throw new IllegalStateException("System under test is shut down");
-
- // TODO: limit a number of retries
- sut.executeAsync(statement.cql(), SystemUnderTest.ConsistencyLevel.QUORUM, statement.bindings())
- .whenComplete((res, t) -> {
- if (t != null)
- executor.schedule(() -> executeAsyncWithRetries(future, statement), 1, TimeUnit.SECONDS);
- else
- future.complete(res);
- });
- }
-
- public void shutdown() throws InterruptedException
- {
- executor.shutdown();
- executor.awaitTermination(30, TimeUnit.SECONDS);
- }
-}
diff --git a/harry-core/src/harry/visitors/MutatingRowVisitor.java b/harry-core/src/harry/visitors/MutatingRowVisitor.java
index 0c5db9a..5a1dfbd 100644
--- a/harry-core/src/harry/visitors/MutatingRowVisitor.java
+++ b/harry-core/src/harry/visitors/MutatingRowVisitor.java
@@ -30,7 +30,7 @@ import harry.operations.Query;
import harry.operations.QueryGenerator;
import harry.util.BitSet;
-public class MutatingRowVisitor implements Operation
+public class MutatingRowVisitor implements OperationExecutor
{
protected final SchemaSpec schema;
protected final OpSelectors.MonotonicClock clock;
@@ -64,37 +64,37 @@ public class MutatingRowVisitor implements Operation
public CompiledStatement insert(long lts, long pd, long cd, long opId)
{
metricReporter.insert();
- long[] vds = descriptorSelector.vds(pd, cd, lts, opId, schema);
+ long[] vds = descriptorSelector.vds(pd, cd, lts, opId, OpSelectors.OperationKind.INSERT, schema);
return WriteHelper.inflateInsert(schema, pd, cd, vds, null, clock.rts(lts));
}
public CompiledStatement insertWithStatics(long lts, long pd, long cd, long opId)
{
metricReporter.insert();
- long[] vds = descriptorSelector.vds(pd, cd, lts, opId, schema);
- long[] sds = descriptorSelector.sds(pd, cd, lts, opId, schema);
+ long[] vds = descriptorSelector.vds(pd, cd, lts, opId, OpSelectors.OperationKind.INSERT_WITH_STATICS, schema);
+ long[] sds = descriptorSelector.sds(pd, cd, lts, opId, OpSelectors.OperationKind.INSERT_WITH_STATICS, schema);
return WriteHelper.inflateInsert(schema, pd, cd, vds, sds, clock.rts(lts));
}
public CompiledStatement update(long lts, long pd, long cd, long opId)
{
metricReporter.insert();
- long[] vds = descriptorSelector.vds(pd, cd, lts, opId, schema);
+ long[] vds = descriptorSelector.vds(pd, cd, lts, opId, OpSelectors.OperationKind.UPDATE, schema);
return WriteHelper.inflateUpdate(schema, pd, cd, vds, null, clock.rts(lts));
}
public CompiledStatement updateWithStatics(long lts, long pd, long cd, long opId)
{
metricReporter.insert();
- long[] vds = descriptorSelector.vds(pd, cd, lts, opId, schema);
- long[] sds = descriptorSelector.sds(pd, cd, lts, opId, schema);
+ long[] vds = descriptorSelector.vds(pd, cd, lts, opId, OpSelectors.OperationKind.UPDATE_WITH_STATICS, schema);
+ long[] sds = descriptorSelector.sds(pd, cd, lts, opId, OpSelectors.OperationKind.UPDATE_WITH_STATICS, schema);
return WriteHelper.inflateUpdate(schema, pd, cd, vds, sds, clock.rts(lts));
}
public CompiledStatement deleteColumn(long lts, long pd, long cd, long opId)
{
metricReporter.columnDelete();
- BitSet columns = descriptorSelector.columnMask(pd, lts, opId);
+ BitSet columns = descriptorSelector.columnMask(pd, lts, opId, OpSelectors.OperationKind.DELETE_COLUMN);
BitSet mask = schema.regularColumnsMask();
return DeleteHelper.deleteColumn(schema, pd, cd, columns, mask, clock.rts(lts));
}
@@ -102,7 +102,7 @@ public class MutatingRowVisitor implements Operation
public CompiledStatement deleteColumnWithStatics(long lts, long pd, long cd, long opId)
{
metricReporter.columnDelete();
- BitSet columns = descriptorSelector.columnMask(pd, lts, opId);
+ BitSet columns = descriptorSelector.columnMask(pd, lts, opId, OpSelectors.OperationKind.DELETE_COLUMN_WITH_STATICS);
BitSet mask = schema.regularAndStaticColumnsMask();
return DeleteHelper.deleteColumn(schema, pd, cd, columns, mask, clock.rts(lts));
}
diff --git a/harry-core/src/harry/visitors/MutatingVisitor.java b/harry-core/src/harry/visitors/MutatingVisitor.java
new file mode 100644
index 0000000..26c7417
--- /dev/null
+++ b/harry-core/src/harry/visitors/MutatingVisitor.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.visitors;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import harry.core.Run;
+import harry.model.OpSelectors;
+import harry.model.sut.SystemUnderTest;
+import harry.operations.CompiledStatement;
+import harry.runner.DataTracker;
+
+public class MutatingVisitor extends GeneratingVisitor
+{
+ private static final Logger logger = LoggerFactory.getLogger(MutatingVisitor.class);
+
+ public MutatingVisitor(Run run,
+ OperationExecutor.RowVisitorFactory rowVisitorFactory)
+ {
+ super(run, new MutatingVisitExecutor(run, rowVisitorFactory.make(run)));
+ }
+
+ public static class MutatingVisitExecutor implements VisitExecutor
+ {
+ private final List<String> statements = new ArrayList<>();
+ private final List<Object> bindings = new ArrayList<>();
+
+ private final List<CompletableFuture<?>> futures = new ArrayList<>();
+
+ protected final ScheduledExecutorService executor = Executors.newScheduledThreadPool(2);
+
+ protected final OpSelectors.DescriptorSelector descriptorSelector;
+ protected final DataTracker tracker;
+ protected final SystemUnderTest sut;
+ protected final OperationExecutor rowVisitor;
+ private final int maxRetries = 10;
+
+ public MutatingVisitExecutor(Run run, OperationExecutor rowVisitor)
+ {
+ this.descriptorSelector = run.descriptorSelector;
+ this.tracker = run.tracker;
+ this.sut = run.sut;
+ this.rowVisitor = rowVisitor;
+ }
+
+ @Override
+ public void beforeLts(long lts, long pd)
+ {
+ tracker.started(lts);
+ }
+
+ @Override
+ public void afterLts(long lts, long pd)
+ {
+ for (CompletableFuture<?> future : futures)
+ {
+ try
+ {
+ future.get();
+ }
+ catch (Throwable t)
+ {
+ throw new IllegalStateException("Couldn't repeat operations within timeout bounds.", t);
+ }
+ }
+ futures.clear();
+ tracker.finished(lts);
+ }
+
+ @Override
+ public void beforeBatch(long lts, long pd, long m)
+ {
+ statements.clear();
+ bindings.clear();
+ }
+
+ @Override
+ public void operation(long lts, long pd, long cd, long m, long opId, OpSelectors.OperationKind opType)
+ {
+ CompiledStatement statement = operationInternal(lts, pd, cd, m, opId, opType);
+
+ statements.add(statement.cql());
+ for (Object binding : statement.bindings())
+ bindings.add(binding);
+ }
+
+ protected CompiledStatement operationInternal(long lts, long pd, long cd, long m, long opId, OpSelectors.OperationKind opType)
+ {
+ return rowVisitor.perform(opType, lts, pd, cd, opId);
+ }
+
+ @Override
+ public void afterBatch(long lts, long pd, long m)
+ {
+ if (statements.isEmpty())
+ {
+ logger.warn("Encountered an empty batch on {}", lts);
+ return;
+ }
+
+ String query = String.join(" ", statements);
+
+ if (statements.size() > 1)
+ query = String.format("BEGIN UNLOGGED BATCH\n%s\nAPPLY BATCH;", query);
+
+ Object[] bindingsArray = new Object[bindings.size()];
+ bindings.toArray(bindingsArray);
+
+ CompletableFuture<Object[][]> future = new CompletableFuture<>();
+ executeAsyncWithRetries(lts, pd, future, new CompiledStatement(query, bindingsArray));
+ futures.add(future);
+
+ statements.clear();
+ bindings.clear();
+ }
+
+ protected void executeAsyncWithRetries(long lts, long pd, CompletableFuture<Object[][]> future, CompiledStatement statement)
+ {
+ executeAsyncWithRetries(lts, pd, future, statement, 0);
+ }
+
+ private void executeAsyncWithRetries(long lts, long pd, CompletableFuture<Object[][]> future, CompiledStatement statement, int retries)
+ {
+ if (sut.isShutdown())
+ throw new IllegalStateException("System under test is shut down");
+
+ if (retries > this.maxRetries)
+ throw new IllegalStateException(String.format("Can not execute statement %s after %d retries", statement, retries));
+
+ sut.executeAsync(statement.cql(), SystemUnderTest.ConsistencyLevel.QUORUM, statement.bindings())
+ .whenComplete((res, t) -> {
+ if (t != null)
+ executor.schedule(() -> executeAsyncWithRetries(lts, pd, future, statement, retries + 1), 1, TimeUnit.SECONDS);
+ else
+ future.complete(res);
+ });
+ }
+
+ public void shutdown() throws InterruptedException
+ {
+ executor.shutdown();
+ executor.awaitTermination(30, TimeUnit.SECONDS);
+ }
+ }
+}
diff --git a/harry-core/src/harry/visitors/Operation.java b/harry-core/src/harry/visitors/OperationExecutor.java
similarity index 97%
rename from harry-core/src/harry/visitors/Operation.java
rename to harry-core/src/harry/visitors/OperationExecutor.java
index 1af21fc..920b0c5 100644
--- a/harry-core/src/harry/visitors/Operation.java
+++ b/harry-core/src/harry/visitors/OperationExecutor.java
@@ -22,11 +22,11 @@ import harry.core.Run;
import harry.model.OpSelectors;
import harry.operations.CompiledStatement;
-public interface Operation
+public interface OperationExecutor
{
interface RowVisitorFactory
{
- Operation make(Run run);
+ OperationExecutor make(Run run);
}
default CompiledStatement perform(OpSelectors.OperationKind op, long lts, long pd, long cd, long opId)
diff --git a/harry-core/src/harry/visitors/ParallelRecentPartitionValidator.java b/harry-core/src/harry/visitors/ParallelRecentValidator.java
similarity index 80%
rename from harry-core/src/harry/visitors/ParallelRecentPartitionValidator.java
rename to harry-core/src/harry/visitors/ParallelRecentValidator.java
index a1688cd..29503d1 100644
--- a/harry-core/src/harry/visitors/ParallelRecentPartitionValidator.java
+++ b/harry-core/src/harry/visitors/ParallelRecentValidator.java
@@ -41,9 +41,9 @@ import harry.model.Model;
import harry.operations.Query;
import harry.operations.QueryGenerator;
-public class ParallelRecentPartitionValidator extends ParallelValidator<ParallelRecentPartitionValidator.State>
+public class ParallelRecentValidator extends ParallelValidator<ParallelRecentValidator.State>
{
- private static final Logger logger = LoggerFactory.getLogger(ParallelRecentPartitionValidator.class);
+ private static final Logger logger = LoggerFactory.getLogger(ParallelRecentValidator.class);
private final int partitionCount;
private final int queries;
@@ -51,9 +51,9 @@ public class ParallelRecentPartitionValidator extends ParallelValidator<Parallel
private final Model model;
private final BufferedWriter validationLog;
- public ParallelRecentPartitionValidator(int partitionCount, int concurrency, int triggerAfter, int queries,
- Run run,
- Model.ModelFactory modelFactory)
+ public ParallelRecentValidator(int partitionCount, int concurrency, int triggerAfter, int queries,
+ Run run,
+ Model.ModelFactory modelFactory)
{
super(concurrency, triggerAfter, run);
this.partitionCount = partitionCount;
@@ -139,7 +139,7 @@ public class ParallelRecentPartitionValidator extends ParallelValidator<Parallel
}
@JsonTypeName("parallel_validate_recent_partitions")
- public static class ParallelRecentPartitionValidatorConfig implements Configuration.PartitionVisitorConfiguration
+ public static class ParallelRecentValidatorConfig implements Configuration.VisitorConfiguration
{
public final int partition_count;
public final int trigger_after;
@@ -149,11 +149,11 @@ public class ParallelRecentPartitionValidator extends ParallelValidator<Parallel
// TODO: make query selector configurable
@JsonCreator
- public ParallelRecentPartitionValidatorConfig(@JsonProperty("partition_count") int partition_count,
- @JsonProperty("concurrency") int concurrency,
- @JsonProperty("trigger_after") int trigger_after,
- @JsonProperty("queries_per_partition") int queries,
- @JsonProperty("model") Configuration.ModelConfiguration model)
+ public ParallelRecentValidatorConfig(@JsonProperty("partition_count") int partition_count,
+ @JsonProperty("concurrency") int concurrency,
+ @JsonProperty("trigger_after") int trigger_after,
+ @JsonProperty("queries_per_partition") int queries,
+ @JsonProperty("model") Configuration.ModelConfiguration model)
{
this.partition_count = partition_count;
this.concurrency = concurrency;
@@ -163,9 +163,9 @@ public class ParallelRecentPartitionValidator extends ParallelValidator<Parallel
}
@Override
- public PartitionVisitor make(Run run)
+ public Visitor make(Run run)
{
- return new ParallelRecentPartitionValidator(partition_count, concurrency, trigger_after, queries, run, modelConfiguration);
+ return new ParallelRecentValidator(partition_count, concurrency, trigger_after, queries, run, modelConfiguration);
}
}
diff --git a/harry-core/src/harry/visitors/ParallelValidator.java b/harry-core/src/harry/visitors/ParallelValidator.java
index 2964eb7..4772b40 100644
--- a/harry-core/src/harry/visitors/ParallelValidator.java
+++ b/harry-core/src/harry/visitors/ParallelValidator.java
@@ -30,7 +30,7 @@ import org.slf4j.LoggerFactory;
import harry.core.Run;
-public abstract class ParallelValidator<T extends ParallelValidator.State> implements PartitionVisitor
+public abstract class ParallelValidator<T extends ParallelValidator.State> implements Visitor
{
private static final Logger logger = LoggerFactory.getLogger(AllPartitionsValidator.class);
@@ -86,7 +86,7 @@ public abstract class ParallelValidator<T extends ParallelValidator.State> imple
}
}
- public void visitPartition(long lts)
+ public void visit(long lts)
{
maxPos.updateAndGet(current -> Math.max(run.pdSelector.positionFor(lts), current));
diff --git a/harry-core/src/harry/visitors/RecentPartitionValidator.java b/harry-core/src/harry/visitors/RecentValidator.java
similarity index 91%
rename from harry-core/src/harry/visitors/RecentPartitionValidator.java
rename to harry-core/src/harry/visitors/RecentValidator.java
index 4bb1e61..d0d09fa 100644
--- a/harry-core/src/harry/visitors/RecentPartitionValidator.java
+++ b/harry-core/src/harry/visitors/RecentValidator.java
@@ -37,10 +37,10 @@ import harry.model.OpSelectors;
import harry.operations.Query;
import harry.operations.QueryGenerator;
-public class RecentPartitionValidator implements PartitionVisitor
+public class RecentValidator implements Visitor
{
private final BufferedWriter validationLog;
- private static final Logger logger = LoggerFactory.getLogger(RecentPartitionValidator.class);
+ private static final Logger logger = LoggerFactory.getLogger(RecentValidator.class);
private final Model model;
private final OpSelectors.PdSelector pdSelector;
@@ -50,11 +50,11 @@ public class RecentPartitionValidator implements PartitionVisitor
private final int triggerAfter;
private final int queries;
- public RecentPartitionValidator(int partitionCount,
- int queries,
- int triggerAfter,
- Run run,
- Model.ModelFactory modelFactory)
+ public RecentValidator(int partitionCount,
+ int queries,
+ int triggerAfter,
+ Run run,
+ Model.ModelFactory modelFactory)
{
this.partitionCount = partitionCount;
this.triggerAfter = triggerAfter;
@@ -103,7 +103,7 @@ public class RecentPartitionValidator implements PartitionVisitor
}
}
- public void visitPartition(long lts)
+ public void visit(long lts)
{
maxPos.updateAndGet(current -> Math.max(pdSelector.positionFor(lts), current));
diff --git a/harry-core/src/harry/visitors/ReplayingVisitor.java b/harry-core/src/harry/visitors/ReplayingVisitor.java
new file mode 100644
index 0000000..1979664
--- /dev/null
+++ b/harry-core/src/harry/visitors/ReplayingVisitor.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.visitors;
+
+import java.util.Arrays;
+
+import harry.core.Run;
+import harry.model.OpSelectors;
+
+public abstract class ReplayingVisitor extends DelegatingVisitor
+{
+ public ReplayingVisitor(VisitExecutor delegate)
+ {
+ super(delegate);
+ }
+
+ public void visit(long lts)
+ {
+ replay(getVisit(lts));
+ }
+
+ public abstract Visit getVisit(long lts);
+
+ public abstract void replayAll(Run run);
+ private void replay(Visit visit)
+ {
+ beforeLts(visit.lts, visit.pd);
+
+ for (Batch batch : visit.operations)
+ {
+ beforeBatch(visit.lts, visit.pd, batch.m);
+ for (Operation operation : batch.operations)
+ operation(visit.lts, visit.pd, operation.cd, batch.m, operation.opId, operation.opType);
+ afterBatch(visit.lts, visit.pd, batch.m);
+ }
+
+ afterLts(visit.lts, visit.pd);
+ }
+
+ public static class Visit
+ {
+ public final long lts;
+ public final long pd;
+ public final Batch[] operations;
+
+ public Visit(long lts, long pd, Batch[] operations)
+ {
+ this.lts = lts;
+ this.pd = pd;
+ this.operations = operations;
+ }
+
+ public String toString()
+ {
+ return "Visit{" +
+ "lts=" + lts +
+ ", pd=" + pd +
+ ", operations=[" + Arrays.toString(operations) +
+ "]}";
+ }
+ }
+
+ public static class Batch
+ {
+ public final long m;
+ public final Operation[] operations;
+
+ public Batch(long m, Operation[] operations)
+ {
+ this.m = m;
+ this.operations = operations;
+ }
+
+ public String toString()
+ {
+ return "Batch{" +
+ "m=" + m +
+ ", operations=[" + Arrays.toString(operations) +
+ "]}";
+ }
+ }
+
+ public static class Operation
+ {
+ public final long cd;
+ public final long opId;
+ public final OpSelectors.OperationKind opType;
+
+ public Operation(long cd, long opId, OpSelectors.OperationKind opType)
+ {
+ this.cd = cd;
+ this.opId = opId;
+ this.opType = opType;
+ }
+
+ public String toString()
+ {
+ return "Operation{" +
+ "cd=" + cd +
+ ", opId=" + opId +
+ ", opType=" + opType +
+ '}';
+ }
+ }
+}
\ No newline at end of file
diff --git a/harry-core/src/harry/visitors/Sampler.java b/harry-core/src/harry/visitors/Sampler.java
index e4088d9..20c9a8c 100644
--- a/harry-core/src/harry/visitors/Sampler.java
+++ b/harry-core/src/harry/visitors/Sampler.java
@@ -35,7 +35,7 @@ import harry.model.OpSelectors;
import harry.model.SelectHelper;
import harry.model.sut.SystemUnderTest;
-public class Sampler implements PartitionVisitor
+public class Sampler implements Visitor
{
private static final Logger logger = LoggerFactory.getLogger(AllPartitionsValidator.class);
@@ -57,7 +57,7 @@ public class Sampler implements PartitionVisitor
this.samplePartitions = samplePartitions;
}
- public void visitPartition(long lts)
+ public void visit(long lts)
{
maxPos.updateAndGet(current -> Math.max(pdSelector.positionFor(lts), current));
@@ -89,7 +89,7 @@ public class Sampler implements PartitionVisitor
}
@JsonTypeName("sampler")
- public static class SamplerConfiguration implements Configuration.PartitionVisitorConfiguration
+ public static class SamplerConfiguration implements Configuration.VisitorConfiguration
{
public final int trigger_after;
public final int sample_partitions;
@@ -102,7 +102,7 @@ public class Sampler implements PartitionVisitor
this.sample_partitions = sample_partitions;
}
- public PartitionVisitor make(Run run)
+ public Visitor make(Run run)
{
return new Sampler(run, trigger_after, sample_partitions);
}
diff --git a/harry-core/src/harry/visitors/SinglePartitionValidator.java b/harry-core/src/harry/visitors/SingleValidator.java
similarity index 86%
rename from harry-core/src/harry/visitors/SinglePartitionValidator.java
rename to harry-core/src/harry/visitors/SingleValidator.java
index 313cff9..0993461 100644
--- a/harry-core/src/harry/visitors/SinglePartitionValidator.java
+++ b/harry-core/src/harry/visitors/SingleValidator.java
@@ -22,17 +22,16 @@ import harry.core.Run;
import harry.model.Model;
import harry.operations.Query;
import harry.operations.QueryGenerator;
-import harry.visitors.PartitionVisitor;
-public class SinglePartitionValidator implements PartitionVisitor
+public class SingleValidator implements Visitor
{
protected final int iterations;
protected final Model model;
protected final QueryGenerator queryGenerator;
protected final Run run;
- public SinglePartitionValidator(int iterations,
- Run run,
- Model.ModelFactory modelFactory)
+ public SingleValidator(int iterations,
+ Run run,
+ Model.ModelFactory modelFactory)
{
this.iterations = iterations;
this.model = modelFactory.make(run);
@@ -45,7 +44,7 @@ public class SinglePartitionValidator implements PartitionVisitor
}
- public void visitPartition(long lts)
+ public void visit(long lts)
{
model.validate(queryGenerator.inflate(lts, 0, Query.QueryKind.SINGLE_PARTITION));
diff --git a/harry-core/src/harry/visitors/PartitionVisitor.java b/harry-core/src/harry/visitors/VisitExecutor.java
similarity index 65%
copy from harry-core/src/harry/visitors/PartitionVisitor.java
copy to harry-core/src/harry/visitors/VisitExecutor.java
index 77de711..94aa1fc 100644
--- a/harry-core/src/harry/visitors/PartitionVisitor.java
+++ b/harry-core/src/harry/visitors/VisitExecutor.java
@@ -18,14 +18,19 @@
package harry.visitors;
-import harry.core.Run;
+import harry.model.OpSelectors;
-public interface PartitionVisitor
+public interface VisitExecutor
{
- void visitPartition(long lts);
- public void shutdown() throws InterruptedException;
- public interface PartitionVisitorFactory
- {
- public PartitionVisitor make(Run run);
- }
-}
\ No newline at end of file
+ public void beforeLts(long lts, long pd);
+
+ public void afterLts(long lts, long pd);
+
+ public void beforeBatch(long lts, long pd, long m);
+
+ public void operation(long lts, long pd, long cd, long m, long opId, OpSelectors.OperationKind kind);
+
+ public void afterBatch(long lts, long pd, long m);
+
+ public default void shutdown() throws InterruptedException {}
+}
diff --git a/harry-core/src/harry/visitors/PartitionVisitor.java b/harry-core/src/harry/visitors/Visitor.java
similarity index 80%
copy from harry-core/src/harry/visitors/PartitionVisitor.java
copy to harry-core/src/harry/visitors/Visitor.java
index 77de711..7cb6111 100644
--- a/harry-core/src/harry/visitors/PartitionVisitor.java
+++ b/harry-core/src/harry/visitors/Visitor.java
@@ -20,12 +20,14 @@ package harry.visitors;
import harry.core.Run;
-public interface PartitionVisitor
+public interface Visitor
{
- void visitPartition(long lts);
- public void shutdown() throws InterruptedException;
- public interface PartitionVisitorFactory
+ void visit(long lts);
+
+ public default void shutdown() throws InterruptedException {}
+
+ public interface VisitorFactory
{
- public PartitionVisitor make(Run run);
+ public Visitor make(Run run);
}
}
\ No newline at end of file
diff --git a/harry-core/test/harry/model/OpSelectorsTest.java b/harry-core/test/harry/model/OpSelectorsTest.java
index 709ba81..0f474f0 100644
--- a/harry-core/test/harry/model/OpSelectorsTest.java
+++ b/harry-core/test/harry/model/OpSelectorsTest.java
@@ -44,9 +44,9 @@ import harry.model.clock.OffsetClock;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
import harry.runner.DataTracker;
-import harry.visitors.MutatingPartitionVisitor;
-import harry.visitors.PartitionVisitor;
-import harry.visitors.Operation;
+import harry.visitors.MutatingVisitor;
+import harry.visitors.Visitor;
+import harry.visitors.OperationExecutor;
import harry.util.BitSet;
public class OpSelectorsTest
@@ -213,8 +213,8 @@ public class OpSelectorsTest
SystemUnderTest.NO_OP,
MetricReporter.NO_OP);
- PartitionVisitor partitionVisitor = new MutatingPartitionVisitor(run,
- (r) -> new Operation()
+ Visitor visitor = new MutatingVisitor(run,
+ (r) -> new OperationExecutor()
{
public CompiledStatement insert(long lts, long pd, long cd, long m)
{
@@ -279,7 +279,7 @@ public class OpSelectorsTest
for (int lts = 0; lts < 1000; lts++)
{
- partitionVisitor.visitPartition(lts);
+ visitor.visit(lts);
}
for (Collection<Long> value : partitionMap.values())
diff --git a/harry-core/test/harry/operations/RelationTest.java b/harry-core/test/harry/operations/RelationTest.java
index e0c6e73..eaa7ddb 100644
--- a/harry-core/test/harry/operations/RelationTest.java
+++ b/harry-core/test/harry/operations/RelationTest.java
@@ -173,7 +173,7 @@ public class RelationTest
throw new RuntimeException("not implemented");
}
- public BitSet columnMask(long pd, long lts, long opId)
+ public BitSet columnMask(long pd, long lts, long opId, OpSelectors.OperationKind opType)
{
throw new RuntimeException("not implemented");
}
diff --git a/harry-integration-backup/test/resources/single_partition_test.yml b/harry-integration-backup/test/resources/single_partition_test.yml
deleted file mode 100644
index 0ebe2aa..0000000
--- a/harry-integration-backup/test/resources/single_partition_test.yml
+++ /dev/null
@@ -1,55 +0,0 @@
-seed: 1
-
-# Default schema provider generates random schema
-schema_provider:
- default: {}
-
-drop_schema: false
-create_schema: true
-truncate_table: false
-
-clock:
- offset:
- offset: 1000
-
-run_time: 10
-run_time_unit: "MINUTES"
-
-system_under_test:
- println: {}
-
-partition_descriptor_selector:
- always_same:
- pd: 12345
-
-clustering_descriptor_selector:
- default:
- modifications_per_lts:
- type: "constant"
- constant: 2
- rows_per_modification:
- type: "constant"
- constant: 2
- operation_kind_weights:
- DELETE_RANGE: 1
- DELETE_SLICE: 1
- DELETE_ROW: 1
- DELETE_COLUMN: 1
- DELETE_PARTITION: 1
- DELETE_COLUMN_WITH_STATICS: 1
- INSERT_WITH_STATICS: 24
- INSERT: 24
- UPDATE_WITH_STATICS: 23
- UPDATE: 23
- column_mask_bitsets: null
- max_partition_size: 100
-
-data_tracker:
- no_op: {}
-
-runner:
- sequential:
- partition_visitors: []
-
-metric_reporter:
- no_op: {}
\ No newline at end of file
diff --git a/harry-integration/src/harry/model/sut/InJVMTokenAwareVisitExecutor.java b/harry-integration/src/harry/model/sut/InJVMTokenAwareVisitExecutor.java
new file mode 100644
index 0000000..ff4ad86
--- /dev/null
+++ b/harry-integration/src/harry/model/sut/InJVMTokenAwareVisitExecutor.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.model.sut;
+
+import java.util.Random;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import harry.core.Configuration;
+import harry.core.Run;
+import harry.ddl.SchemaSpec;
+import harry.operations.CompiledStatement;
+import harry.visitors.GeneratingVisitor;
+import harry.visitors.LoggingVisitor;
+import harry.visitors.OperationExecutor;
+import harry.visitors.Visitor;
+
+public class InJVMTokenAwareVisitExecutor extends LoggingVisitor.LoggingVisitorExecutor
+{
+ public static void init()
+ {
+ Configuration.registerSubtypes(Configuation.class);
+ }
+
+ private final InJvmSut sut;
+ private final SystemUnderTest.ConsistencyLevel cl;
+ private final SchemaSpec schema;
+ private final int maxRetries = 10;
+
+ public InJVMTokenAwareVisitExecutor(Run run,
+ OperationExecutor.RowVisitorFactory rowVisitorFactory,
+ SystemUnderTest.ConsistencyLevel cl)
+ {
+ super(run, rowVisitorFactory.make(run));
+ this.sut = (InJvmSut) run.sut;
+ this.schema = run.schemaSpec;
+ this.cl = cl;
+ }
+
+ @Override
+ protected void executeAsyncWithRetries(long lts, long pd, CompletableFuture<Object[][]> future, CompiledStatement statement)
+ {
+ executeAsyncWithRetries(lts, pd, future, statement, 0);
+ }
+
+ private void executeAsyncWithRetries(long lts, long pd, CompletableFuture<Object[][]> future, CompiledStatement statement, int retries)
+ {
+ if (sut.isShutdown())
+ throw new IllegalStateException("System under test is shut down");
+
+ if (retries > this.maxRetries)
+ throw new IllegalStateException(String.format("Can not execute statement %s after %d retries", statement, retries));
+
+ Object[] partitionKey = schema.inflatePartitionKey(pd);
+ int[] replicas = sut.getReplicasFor(partitionKey, schema.keyspace, schema.table);
+ // TODO: find a better source of entropy
+ int replica = replicas[new Random(lts).nextInt(replicas.length)];
+ if (cl == SystemUnderTest.ConsistencyLevel.NODE_LOCAL)
+ {
+ future.complete(sut.cluster.get(replica).executeInternal(statement.cql(), statement.bindings()));
+ }
+ else
+ {
+ CompletableFuture.supplyAsync(() -> sut.cluster.coordinator(replica).execute(statement.cql(), InJvmSut.toApiCl(cl), statement.bindings()), executor)
+ .whenComplete((res, t) ->
+ {
+ if (t != null)
+ executor.schedule(() -> executeAsyncWithRetries(lts, pd, future, statement, retries + 1), 1, TimeUnit.SECONDS);
+ else
+ future.complete(res);
+ });
+ }
+ }
+
+ @JsonTypeName("in_jvm_token_aware")
+ public static class Configuation implements Configuration.VisitorConfiguration
+ {
+ public final Configuration.RowVisitorConfiguration row_visitor;
+ public final SystemUnderTest.ConsistencyLevel consistency_level;
+
+ @JsonCreator
+ public Configuation(@JsonProperty("row_visitor") Configuration.RowVisitorConfiguration rowVisitor,
+ @JsonProperty("consistency_level") SystemUnderTest.ConsistencyLevel consistencyLevel)
+ {
+ this.row_visitor = rowVisitor;
+ this.consistency_level = consistencyLevel;
+ }
+
+ @Override
+ public Visitor make(Run run)
+ {
+ return new GeneratingVisitor(run, new InJVMTokenAwareVisitExecutor(run, row_visitor, consistency_level));
+ }
+ }
+}
\ No newline at end of file
diff --git a/harry-integration/src/harry/model/sut/InJvmSut.java b/harry-integration/src/harry/model/sut/InJvmSut.java
index 930c4d4..f758ca5 100644
--- a/harry-integration/src/harry/model/sut/InJvmSut.java
+++ b/harry-integration/src/harry/model/sut/InJvmSut.java
@@ -20,7 +20,10 @@ package harry.model.sut;
import java.io.File;
import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
import java.util.function.Consumer;
+import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -29,9 +32,15 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import harry.core.Configuration;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
+import org.apache.cassandra.db.marshal.CompositeType;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.apache.cassandra.locator.EndpointsForToken;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.ByteBufferUtil;
public class InJvmSut extends InJvmSutBase<IInvokableInstance, Cluster>
{
@@ -83,4 +92,17 @@ public class InJvmSut extends InJvmSutBase<IInvokableInstance, Cluster>
return new InJvmSut(cluster);
}
}
+
+ public int[] getReplicasFor(Object[] partitionKey, String keyspace, String table)
+ {
+ return cluster.get(1).appliesOnInstance((Object[] pk, String ks) ->
+ {
+ String pkString = Arrays.asList(pk).stream().map(Object::toString).collect(Collectors.joining(":"));
+ EndpointsForToken endpoints = StorageService.instance.getNaturalReplicasForToken(ks, table, pkString);
+ int[] nodes = new int[endpoints.size()];
+ for (int i = 0; i < endpoints.size(); i++)
+ nodes[i] = endpoints.get(i).endpoint().address.getAddress()[3];
+ return nodes;
+ }).apply(partitionKey, keyspace);
+ }
}
\ No newline at end of file
diff --git a/harry-core/src/harry/visitors/FaultInjectingPartitionVisitor.java b/harry-integration/src/harry/runner/FaultInjectingVisitor.java
similarity index 75%
rename from harry-core/src/harry/visitors/FaultInjectingPartitionVisitor.java
rename to harry-integration/src/harry/runner/FaultInjectingVisitor.java
index 6a03d35..4cca1b2 100644
--- a/harry-core/src/harry/visitors/FaultInjectingPartitionVisitor.java
+++ b/harry-integration/src/harry/runner/FaultInjectingVisitor.java
@@ -16,9 +16,11 @@
* limitations under the License.
*/
-package harry.visitors;
+package harry.runner;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -29,35 +31,39 @@ import harry.core.Configuration;
import harry.core.Run;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
+import harry.visitors.LoggingVisitor;
+import harry.visitors.OperationExecutor;
+import harry.visitors.Visitor;
-public class FaultInjectingPartitionVisitor extends LoggingPartitionVisitor
+public class FaultInjectingVisitor extends LoggingVisitor
{
public static void init()
{
- Configuration.registerSubtypes(FaultInjectingPartitionVisitorConfiguration.class);
+ Configuration.registerSubtypes(FaultInjectingVisitorConfiguration.class);
}
@JsonTypeName("fault_injecting")
- public static class FaultInjectingPartitionVisitorConfiguration extends Configuration.MutatingPartitionVisitorConfiguation
+ public static class FaultInjectingVisitorConfiguration extends Configuration.MutatingVisitorConfiguation
{
@JsonCreator
- public FaultInjectingPartitionVisitorConfiguration(@JsonProperty("row_visitor") Configuration.RowVisitorConfiguration row_visitor)
+ public FaultInjectingVisitorConfiguration(@JsonProperty("row_visitor") Configuration.RowVisitorConfiguration row_visitor)
{
super(row_visitor);
}
@Override
- public PartitionVisitor make(Run run)
+ public Visitor make(Run run)
{
- return new FaultInjectingPartitionVisitor(run, row_visitor);
+ return new FaultInjectingVisitor(run, row_visitor);
}
}
private final AtomicInteger cnt = new AtomicInteger();
private final SystemUnderTest.FaultInjectingSut sut;
+ protected final ScheduledExecutorService executor = Executors.newScheduledThreadPool(2);
- public FaultInjectingPartitionVisitor(Run run, Operation.RowVisitorFactory rowVisitorFactory)
+ public FaultInjectingVisitor(Run run, OperationExecutor.RowVisitorFactory rowVisitorFactory)
{
super(run, rowVisitorFactory);
this.sut = (SystemUnderTest.FaultInjectingSut) run.sut;
diff --git a/harry-integration/src/harry/runner/RepairingLocalStateValidator.java b/harry-integration/src/harry/runner/RepairingLocalStateValidator.java
index 41926f0..fe811e5 100644
--- a/harry-integration/src/harry/runner/RepairingLocalStateValidator.java
+++ b/harry-integration/src/harry/runner/RepairingLocalStateValidator.java
@@ -19,6 +19,7 @@
package harry.runner;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonCreator;
@@ -34,7 +35,7 @@ import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
import harry.operations.Query;
import harry.visitors.AllPartitionsValidator;
-import harry.visitors.PartitionVisitor;
+import harry.visitors.Visitor;
import static harry.model.SelectHelper.resultSetToRow;
@@ -56,19 +57,22 @@ public class RepairingLocalStateValidator extends AllPartitionsValidator
}
@Override
- public void visitPartition(long lts)
+ public void visit(long lts)
{
if (lts > 0 && lts % triggerAfter == 0)
{
System.out.println("Starting repair...");
- inJvmSut.cluster().get(1).nodetool("repair", "--full");
+
+ inJvmSut.cluster().stream().forEach((instance) -> {
+ instance.nodetool("repair", "--full");
+ });
System.out.println("Validating partitions...");
- validateAllPartitions(executor, concurrency);
+ super.visit(lts);
}
}
@JsonTypeName("repair_and_validate_local_states")
- public static class RepairingLocalStateValidatorConfiguration implements Configuration.PartitionVisitorConfiguration
+ public static class RepairingLocalStateValidatorConfiguration implements Configuration.VisitorConfiguration
{
private final int concurrency;
private final int trigger_after;
@@ -84,7 +88,7 @@ public class RepairingLocalStateValidator extends AllPartitionsValidator
this.modelConfiguration = model;
}
- public PartitionVisitor make(Run run)
+ public Visitor make(Run run)
{
return new RepairingLocalStateValidator(concurrency, trigger_after, run, modelConfiguration);
}
@@ -106,9 +110,9 @@ public class RepairingLocalStateValidator extends AllPartitionsValidator
public void validate(Query query)
{
CompiledStatement compiled = query.toSelectStatement();
- for (int i = 1; i <= inJvmSut.cluster.size(); i++)
+ int[] replicas = inJvmSut.getReplicasFor(schemaSpec.inflatePartitionKey(query.pd), schemaSpec.keyspace, schemaSpec.table);
+ for (int node : replicas)
{
- int node = i;
validate(() -> {
Object[][] objects = inJvmSut.execute(compiled.cql(),
SystemUnderTest.ConsistencyLevel.NODE_LOCAL,
diff --git a/harry-integration/src/harry/runner/TrivialShrinker.java b/harry-integration/src/harry/runner/TrivialShrinker.java
index 23bdbcc..2879e0e 100644
--- a/harry-integration/src/harry/runner/TrivialShrinker.java
+++ b/harry-integration/src/harry/runner/TrivialShrinker.java
@@ -28,9 +28,9 @@ import java.util.function.Predicate;
import harry.core.Configuration;
import harry.core.Run;
-import harry.visitors.AbstractPartitionVisitor;
-import harry.visitors.PartitionVisitor;
-import harry.visitors.SkippingPartitionVisitor;
+import harry.visitors.DelegatingVisitor;
+import harry.visitors.Visitor;
+import harry.visitors.SkippingVisitor;
/**
* A most trivial imaginable shrinker: attempts to skip partitions and/or logical timestamps to see if the
@@ -62,15 +62,16 @@ public class TrivialShrinker
Run run = configuration.createRun();
Configuration.SequentialRunnerConfig config = (Configuration.SequentialRunnerConfig) configuration.runner;
- List<PartitionVisitor> visitors = new ArrayList<>();
- for (Configuration.PartitionVisitorConfiguration factory : config.partition_visitor_factories)
+ List<Visitor> visitors = new ArrayList<>();
+ for (Configuration.VisitorConfiguration factory : config.visitor_factories)
{
- PartitionVisitor visitor = factory.make(run);
- if (visitor instanceof AbstractPartitionVisitor)
+ Visitor visitor = factory.make(run);
+ if (visitor instanceof DelegatingVisitor)
{
- visitors.add(new SkippingPartitionVisitor((AbstractPartitionVisitor) visitor,
- ltsToSkip,
- pdsToSkip));
+ visitors.add(new SkippingVisitor(visitor,
+ (lts) -> run.pdSelector.pd(lts, run.schemaSpec),
+ ltsToSkip,
+ pdsToSkip));
}
else
{
@@ -159,13 +160,13 @@ public class TrivialShrinker
}
}
- public static void runOnce(Run run, List<PartitionVisitor> visitors, long maxLts)
+ public static void runOnce(Run run, List<Visitor> visitors, long maxLts)
{
for (long lts = 0; lts <= maxLts; lts++)
{
- for (PartitionVisitor visitor : visitors)
+ for (Visitor visitor : visitors)
{
- visitor.visitPartition(lts);
+ visitor.visit(lts);
}
}
}
@@ -182,4 +183,4 @@ public class TrivialShrinker
}
return s.substring(0, s.length() - 1);
}
-}
+}
\ No newline at end of file
diff --git a/harry-integration/src/harry/visitors/SkippingPartitionVisitor.java b/harry-integration/src/harry/visitors/SkippingPartitionVisitor.java
deleted file mode 100644
index fa910cb..0000000
--- a/harry-integration/src/harry/visitors/SkippingPartitionVisitor.java
+++ /dev/null
@@ -1,53 +0,0 @@
-package harry.visitors;
-
-import java.util.Set;
-
-public class SkippingPartitionVisitor extends AbstractPartitionVisitor
-{
- private final AbstractPartitionVisitor delegate;
- private final Set<Long> ltsToSkip;
- private final Set<Long> pdsToSkip;
-
- public SkippingPartitionVisitor(AbstractPartitionVisitor delegate,
- Set<Long> ltsToSkip,
- Set<Long> pdsToSkip)
- {
- super(delegate);
- this.delegate = delegate;
- this.ltsToSkip = ltsToSkip;
- this.pdsToSkip = pdsToSkip;
- }
-
- protected void beforeLts(long lts, long pd)
- {
- delegate.beforeLts(lts, pd);
- }
-
- protected void afterLts(long lts, long pd)
- {
- delegate.afterLts(lts, pd);
- }
-
- protected void beforeBatch(long lts, long pd, long m)
- {
- delegate.beforeBatch(lts, pd, m);
- }
-
- protected void operation(long lts, long pd, long cd, long m, long opId)
- {
- if (pdsToSkip.contains(pd) || ltsToSkip.contains(lts))
- return;
-
- delegate.operation(lts, pd, cd, m, opId);
- }
-
- protected void afterBatch(long lts, long pd, long m)
- {
- delegate.afterBatch(lts, pd, m);
- }
-
- public void shutdown() throws InterruptedException
- {
- delegate.shutdown();
- }
-}
diff --git a/harry-core/src/harry/visitors/PartitionVisitor.java b/harry-integration/src/harry/visitors/SkippingVisitor.java
similarity index 51%
rename from harry-core/src/harry/visitors/PartitionVisitor.java
rename to harry-integration/src/harry/visitors/SkippingVisitor.java
index 77de711..8da0a89 100644
--- a/harry-core/src/harry/visitors/PartitionVisitor.java
+++ b/harry-integration/src/harry/visitors/SkippingVisitor.java
@@ -18,14 +18,36 @@
package harry.visitors;
-import harry.core.Run;
+import java.util.Set;
-public interface PartitionVisitor
+public class SkippingVisitor implements Visitor
{
- void visitPartition(long lts);
- public void shutdown() throws InterruptedException;
- public interface PartitionVisitorFactory
+ private final Set<Long> ltsToSkip;
+ private final Set<Long> pdsToSkip;
+ private final LtsToPd ltsToPd;
+ private final Visitor delegate;
+
+ public SkippingVisitor(Visitor delegate,
+ LtsToPd ltsToPd,
+ Set<Long> ltsToSkip,
+ Set<Long> pdsToSkip)
+ {
+ this.delegate = delegate;
+ this.ltsToSkip = ltsToSkip;
+ this.pdsToSkip = pdsToSkip;
+ this.ltsToPd = ltsToPd;
+ }
+
+ public void visit(long lts)
+ {
+ if (ltsToSkip.contains(lts) || pdsToSkip.contains(ltsToPd.convert(lts)))
+ return;
+
+ delegate.visit(lts);
+ }
+
+ public static interface LtsToPd
{
- public PartitionVisitor make(Run run);
+ public long convert(long lts);
}
-}
\ No newline at end of file
+}
diff --git a/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java b/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
index b819eb5..33463b8 100644
--- a/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
+++ b/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
@@ -1,21 +1,3 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
package harry.generators;
import java.util.Random;
@@ -32,10 +14,10 @@ import harry.generators.distribution.Distribution;
import harry.model.NoOpChecker;
import harry.model.OpSelectors;
import harry.model.sut.SystemUnderTest;
-import harry.visitors.MutatingPartitionVisitor;
+import harry.visitors.MutatingVisitor;
import harry.visitors.MutatingRowVisitor;
-import harry.visitors.PartitionVisitor;
-import harry.visitors.SinglePartitionValidator;
+import harry.visitors.Visitor;
+import harry.visitors.SingleValidator;
import harry.util.TestRunner;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.UntypedResultSet;
@@ -110,16 +92,16 @@ public class DataGeneratorsIntegrationTest extends CQLTester
.build()
.createRun();
- PartitionVisitor visitor = new MutatingPartitionVisitor(run, MutatingRowVisitor::new);
+ Visitor visitor = new MutatingVisitor(run, MutatingRowVisitor::new);
for (int lts = 0; lts < 100; lts++)
- visitor.visitPartition(lts);
+ visitor.visit(lts);
}
Run run = builder.build()
.createRun();
- PartitionVisitor visitor = new SinglePartitionValidator(100, run, NoOpChecker::new);
+ Visitor visitor = new SingleValidator(100, run, NoOpChecker::new);
for (int lts = 0; lts < 100; lts++)
- visitor.visitPartition(lts);
+ visitor.visit(lts);
});
diff --git a/harry-integration/test/harry/model/HistoryBuilderIntegrationTest.java b/harry-integration/test/harry/model/HistoryBuilderIntegrationTest.java
new file mode 100644
index 0000000..6c0ee24
--- /dev/null
+++ b/harry-integration/test/harry/model/HistoryBuilderIntegrationTest.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.model;
+
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+import java.util.function.Supplier;
+
+import org.junit.Test;
+
+import harry.core.Configuration;
+import harry.core.Run;
+import harry.ddl.SchemaGenerators;
+import harry.ddl.SchemaSpec;
+import harry.dsl.HistoryBuilder;
+import harry.operations.CompiledStatement;
+import harry.operations.Query;
+import harry.operations.QueryGenerator;
+import harry.reconciler.Reconciler;
+import harry.util.TestRunner;
+import harry.visitors.MutatingVisitor;
+import harry.visitors.MutatingRowVisitor;
+import harry.visitors.ReplayingVisitor;
+
+public class HistoryBuilderIntegrationTest extends ModelTestBase
+{
+ public Configuration.ConfigurationBuilder configuration(long seed, SchemaSpec schema)
+ {
+ return super.configuration(seed, schema)
+ .setPartitionDescriptorSelector((ignore) -> new HistoryBuilder.PdSelector())
+ // TODO: ideally, we want a custom/tailored clustering descriptor selector
+ .setClusteringDescriptorSelector((builder) -> builder.setNumberOfModificationsDistribution(new Configuration.ConstantDistributionConfig(100_000))
+ .setRowsPerModificationDistribution(new Configuration.ConstantDistributionConfig(100_000)));
+ }
+
+ @Test
+ public void simpleDSLTest()
+ {
+ Supplier<SchemaSpec> supplier = SchemaGenerators.progression(SchemaGenerators.DEFAULT_SWITCH_AFTER);
+ for (int i = 0; i < SchemaGenerators.DEFAULT_RUNS; i++)
+ {
+ SchemaSpec schema = supplier.get();
+ Configuration config = configuration(i, schema).build();
+
+ Run run = config.createRun();
+ beforeEach();
+ run.sut.schemaChange(schema.compile().cql());
+ HistoryBuilder history = new HistoryBuilder(run);
+
+ Set<Long> pds = new HashSet<>();
+
+ for (int j = 0; j < 5; j++)
+ {
+ history.nextPartition()
+ .simultaneously()
+ .batch()
+ .insert()
+ .delete()
+ .finish()
+ .insert()
+ .finish()
+ .nextPartition()
+ .sequentially()
+ .randomOrder()
+ .batch()
+ .insert()
+ .delete()
+ .finish()
+ .updates(5)
+ .partitionDelete()
+ .finish()
+ .nextPartition()
+ .sequentially()
+ .randomOrder()
+ .batch()
+ .insert()
+ .delete()
+ .finish()
+ .updates(10)
+ .partitionDelete()
+ .finish();
+
+ ReplayingVisitor visitor = history.visitor(new MutatingVisitor.MutatingVisitExecutor(run,
+ new MutatingRowVisitor(run)
+ {
+ public CompiledStatement perform(OpSelectors.OperationKind op, long lts, long pd, long cd, long opId)
+ {
+ pds.add(pd);
+ return super.perform(op, lts, pd, cd, opId);
+ }
+ }));
+
+ visitor.replayAll(run);
+
+ Model model = new QuiescentChecker(run, new Reconciler(run,
+ history::visitor));
+ QueryGenerator.TypedQueryGenerator queryGenerator = new QueryGenerator.TypedQueryGenerator(run);
+ for (Long pd : pds)
+ {
+ model.validate(Query.selectPartition(run.schemaSpec,
+ pd,
+ false));
+
+ int lts = new Random().nextInt((int) run.clock.maxLts());
+ for (int k = 0; k < 3; k++)
+ queryGenerator.inflate(lts, k);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testHistoryBuilder() throws Throwable
+ {
+ Supplier<SchemaSpec> supplier = SchemaGenerators.progression(SchemaGenerators.DEFAULT_SWITCH_AFTER);
+ for (int i = 0; i < SchemaGenerators.DEFAULT_RUNS; i++)
+ {
+ SchemaSpec schema = supplier.get();
+ Configuration config = configuration(1L, schema).build();
+ Run run = config.createRun();
+
+ beforeEach();
+ run.sut.schemaChange(schema.compile().cql());
+
+ TestRunner.test((rng) -> {
+ HistoryBuilderTest.Counts counts = new HistoryBuilderTest.Counts();
+ counts.randomOrder = rng.nextBoolean();
+ counts.simultaneously = rng.nextBoolean();
+ counts.partitionDeletion = rng.nextInt(1, 10);
+ counts.update = rng.nextInt(1, 10);
+ counts.insert = rng.nextInt(1, 10);
+ counts.delete = rng.nextInt(1, 10);
+ counts.rangeDelete = rng.nextInt(1, 10);
+ counts.sliceDelete = rng.nextInt(1, 10);
+ counts.columnDelete = rng.nextInt(1, 10);
+ return counts;
+ },
+ () -> new HistoryBuilder(run),
+ (sut, counts) -> {
+ counts.apply(sut);
+ return sut;
+ },
+ (sut) -> {
+ Set<Long> pds = new HashSet<>();
+ ReplayingVisitor visitor = sut.visitor(new MutatingVisitor.MutatingVisitExecutor(run,
+ new MutatingRowVisitor(run)
+ {
+ public CompiledStatement perform(OpSelectors.OperationKind op, long lts, long pd, long cd, long opId)
+ {
+ pds.add(pd);
+ return super.perform(op, lts, pd, cd, opId);
+ }
+ }));
+
+ visitor.replayAll(run);
+
+ Model model = new QuiescentChecker(run, new Reconciler(run,
+ sut::visitor));
+ QueryGenerator.TypedQueryGenerator queryGenerator = new QueryGenerator.TypedQueryGenerator(run);
+ for (Long pd : pds)
+ {
+ model.validate(Query.selectPartition(run.schemaSpec,
+ pd,
+ false));
+ model.validate(Query.selectPartition(run.schemaSpec,
+ pd,
+ true));
+ int lts = new Random().nextInt((int) run.clock.maxLts());
+ for (int k = 0; k < 3; k++)
+ queryGenerator.inflate(lts, k);
+ }
+ });
+ }
+ }
+
+ Configuration.ModelConfiguration modelConfiguration()
+ {
+ return new Configuration.QuiescentCheckerConfig();
+ }
+}
\ No newline at end of file
diff --git a/harry-integration/test/harry/model/HistoryBuilderTest.java b/harry-integration/test/harry/model/HistoryBuilderTest.java
new file mode 100644
index 0000000..5ccca60
--- /dev/null
+++ b/harry-integration/test/harry/model/HistoryBuilderTest.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.model;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import harry.core.Configuration;
+import harry.core.Run;
+import harry.ddl.SchemaSpec;
+import harry.dsl.HistoryBuilder;
+import harry.util.TestRunner;
+import harry.visitors.ReplayingVisitor;
+import harry.visitors.VisitExecutor;
+
+public class HistoryBuilderTest
+{
+ @Test
+ public void testHistoryBuilder() throws Throwable
+ {
+ SchemaSpec schema = MockSchema.tbl1;
+
+ Configuration config = IntegrationTestBase.sharedConfiguration(1, schema)
+ .setPartitionDescriptorSelector((ignore) -> new HistoryBuilder.PdSelector())
+ .setClusteringDescriptorSelector((builder) -> builder.setNumberOfModificationsDistribution(new Configuration.ConstantDistributionConfig(100_000))
+ .setRowsPerModificationDistribution(new Configuration.ConstantDistributionConfig(100_000)))
+ .build();
+
+ Run run = config.createRun();
+ TestRunner.test((rng) -> {
+ Counts counts = new Counts();
+ counts.randomOrder = rng.nextBoolean();
+ counts.simultaneously = rng.nextBoolean();
+ counts.partitionDeletion = rng.nextInt(1, 10);
+ counts.update = rng.nextInt(1, 10);
+ counts.insert = rng.nextInt(1, 10);
+ counts.delete = rng.nextInt(1, 10);
+ counts.rangeDelete = rng.nextInt(1, 10);
+ counts.sliceDelete = rng.nextInt(1, 10);
+ counts.columnDelete = rng.nextInt(1, 10);
+ return counts;
+ },
+ () -> new ArrayList<Counts>(),
+ () -> new HistoryBuilder(run),
+ (model, counts) -> {
+ model.add(counts);
+ return model;
+ },
+ (sut, counts) -> {
+ counts.apply(sut);
+ return sut;
+ },
+ (model, sut) -> {
+ Iterator<Counts> iter = model.iterator();
+ ReplayingVisitor visitor = sut.visitor(new VisitExecutor()
+ {
+ Counts current = iter.next();
+ long lastPd = Long.MIN_VALUE;
+
+ public void beforeLts(long lts, long pd)
+ {
+ if (lastPd == Long.MIN_VALUE)
+ {
+ lastPd = pd;
+ return;
+ }
+
+ if (current.allDone())
+ {
+ Assert.assertNotEquals("Should have switched partition after finishing all operations",
+ pd, lastPd);
+ Assert.assertTrue("System under test still has operations, while model expects none",
+ iter.hasNext());
+ current = iter.next();
+ lastPd = pd;
+ }
+ }
+
+ public void operation(long lts, long pd, long cd, long m, long opId, OpSelectors.OperationKind kind)
+ {
+ switch (kind)
+ {
+ case UPDATE:
+ case UPDATE_WITH_STATICS:
+ current.update--;
+ break;
+ case INSERT:
+ case INSERT_WITH_STATICS:
+ current.insert--;
+ break;
+ case DELETE_PARTITION:
+ current.partitionDeletion--;
+ break;
+ case DELETE_ROW:
+ current.delete--;
+ break;
+ case DELETE_RANGE:
+ current.rangeDelete--;
+ break;
+ case DELETE_SLICE:
+ current.sliceDelete--;
+ break;
+ case DELETE_COLUMN:
+ case DELETE_COLUMN_WITH_STATICS:
+ current.columnDelete--;
+ break;
+ }
+ }
+
+ public void afterLts(long lts, long pd){}
+ public void beforeBatch(long lts, long pd, long m){}
+ public void afterBatch(long lts, long pd, long m){}
+ });
+ visitor.replayAll(run);
+ for (Counts counts : model)
+ Assert.assertTrue(counts.toString(), counts.allDone());
+ });
+ }
+
+ public static class Counts
+ {
+ boolean randomOrder;
+ boolean simultaneously;
+ int partitionDeletion;
+ int update;
+ int insert;
+ int delete;
+ int rangeDelete;
+ int sliceDelete;
+ int columnDelete;
+
+ public void apply(HistoryBuilder sut)
+ {
+ HistoryBuilder.PartitionBuilder builder = sut.nextPartition();
+ if (randomOrder)
+ builder.randomOrder();
+ else
+ builder.strictOrder();
+
+ if (simultaneously)
+ builder.simultaneously();
+ else
+ builder.sequentially();
+
+ builder.deletes(delete);
+ builder.inserts(insert);
+ builder.updates(update);
+ builder.partitionDeletions(partitionDeletion);
+ builder.rangeDeletes(rangeDelete);
+ builder.sliceDeletes(sliceDelete);
+ builder.columnDeletes(columnDelete);
+
+ builder.finish();
+ }
+ boolean allDone()
+ {
+ return partitionDeletion == 0 &&
+ update == 0 &&
+ insert == 0 &&
+ delete == 0 &&
+ rangeDelete == 0 &&
+ sliceDelete == 0 &&
+ columnDelete == 0;
+ }
+ }
+
+}
diff --git a/harry-integration/test/harry/model/InJVMTokenAwareExecutorTest.java b/harry-integration/test/harry/model/InJVMTokenAwareExecutorTest.java
new file mode 100644
index 0000000..0f28005
--- /dev/null
+++ b/harry-integration/test/harry/model/InJVMTokenAwareExecutorTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.model;
+
+import java.util.function.Supplier;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import harry.core.Configuration;
+import harry.core.Run;
+import harry.ddl.SchemaGenerators;
+import harry.ddl.SchemaSpec;
+import harry.model.sut.InJVMTokenAwareVisitExecutor;
+import harry.model.sut.InJvmSut;
+import harry.model.sut.SystemUnderTest;
+import harry.runner.RepairingLocalStateValidator;
+import harry.visitors.GeneratingVisitor;
+import harry.visitors.Visitor;
+import org.apache.cassandra.distributed.Cluster;
+import org.apache.cassandra.distributed.api.Feature;
+
+public class InJVMTokenAwareExecutorTest extends IntegrationTestBase
+{
+ @BeforeClass
+ public static void before() throws Throwable
+ {
+ cluster = init(Cluster.build()
+ .withNodes(5)
+ .withConfig((cfg) -> cfg.with(Feature.GOSSIP, Feature.NETWORK))
+ .start());
+ sut = new InJvmSut(cluster, 1);
+ }
+
+ @Override
+ @Before
+ public void beforeEach()
+ {
+ cluster.schemaChange("DROP KEYSPACE IF EXISTS harry");
+ cluster.schemaChange("CREATE KEYSPACE harry WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};");
+ }
+
+ @Test
+ public void testRepair()
+ {
+ Supplier<SchemaSpec> schemaGen = SchemaGenerators.progression(1);
+ for (int cnt = 0; cnt < SchemaGenerators.DEFAULT_RUNS; cnt++)
+ {
+ SchemaSpec schema = schemaGen.get();
+ Configuration.ConfigurationBuilder builder = sharedConfiguration(cnt, schema);
+
+ Configuration configuration = builder.build();
+ Run run = configuration.createRun();
+ run.sut.schemaChange(run.schemaSpec.compile().cql());
+
+ Visitor visitor = new GeneratingVisitor(run, new InJVMTokenAwareVisitExecutor(run,
+ new Configuration.MutatingRowVisitorConfiguration(),
+ SystemUnderTest.ConsistencyLevel.NODE_LOCAL));
+
+ OpSelectors.MonotonicClock clock = run.clock;
+ long maxPd = 0;
+ for (int i = 0; i < 10000; i++)
+ {
+ long lts = clock.nextLts();
+ visitor.visit(lts);
+ maxPd = Math.max(maxPd, run.pdSelector.positionFor(lts));
+ }
+
+ RepairingLocalStateValidator validator = new RepairingLocalStateValidator(5, 1, run, new Configuration.QuiescentCheckerConfig());
+ validator.visit(clock.maxLts());
+ }
+
+ }
+}
diff --git a/harry-integration/test/harry/model/ModelTestBase.java b/harry-integration/test/harry/model/ModelTestBase.java
index 70b9fb3..576a291 100644
--- a/harry-integration/test/harry/model/ModelTestBase.java
+++ b/harry-integration/test/harry/model/ModelTestBase.java
@@ -28,11 +28,11 @@ import harry.core.Configuration;
import harry.core.Run;
import harry.ddl.SchemaGenerators;
import harry.ddl.SchemaSpec;
-import harry.visitors.LoggingPartitionVisitor;
+import harry.visitors.LoggingVisitor;
import harry.visitors.MutatingRowVisitor;
-import harry.visitors.PartitionVisitor;
+import harry.visitors.Visitor;
import harry.runner.Runner;
-import harry.visitors.SinglePartitionValidator;
+import harry.visitors.SingleValidator;
public abstract class ModelTestBase extends IntegrationTestBase
{
@@ -58,7 +58,7 @@ public abstract class ModelTestBase extends IntegrationTestBase
.setRunTime(1, TimeUnit.MINUTES)
.setCreateSchema(false)
.setDropSchema(false)
- .setRunner(new Configuration.SequentialRunnerConfig(Arrays.asList(new Configuration.LoggingPartitionVisitorConfiguration(new Configuration.MutatingRowVisitorConfiguration()),
+ .setRunner(new Configuration.SequentialRunnerConfig(Arrays.asList(new Configuration.LoggingVisitorConfiguration(new Configuration.MutatingRowVisitorConfiguration()),
new Configuration.RecentPartitionsValidatorConfiguration(10, 10, 1, factory::make),
new Configuration.AllPartitionsValidatorConfiguration(10, 10, factory::make))));
Runner runner = builder.build().createRunner();
@@ -83,9 +83,9 @@ public abstract class ModelTestBase extends IntegrationTestBase
abstract Configuration.ModelConfiguration modelConfiguration();
- protected PartitionVisitor validator(Run run)
+ protected Visitor validator(Run run)
{
- return new SinglePartitionValidator(100, run , modelConfiguration());
+ return new SingleValidator(100, run , modelConfiguration());
}
public Configuration.ConfigurationBuilder configuration(long seed, SchemaSpec schema)
@@ -103,16 +103,16 @@ public abstract class ModelTestBase extends IntegrationTestBase
System.out.println(run.schemaSpec.compile().cql());
OpSelectors.MonotonicClock clock = run.clock;
- PartitionVisitor validator = validator(run);
- PartitionVisitor partitionVisitor = new LoggingPartitionVisitor(run, MutatingRowVisitor::new);
+ Visitor validator = validator(run);
+ Visitor visitor = new LoggingVisitor(run, MutatingRowVisitor::new);
for (int i = 0; i < 20000; i++)
{
long lts = clock.nextLts();
- partitionVisitor.visitPartition(lts);
+ visitor.visit(lts);
}
- validator.visitPartition(0);
+ validator.visit(0);
if (!corrupt.apply(run))
{
@@ -122,7 +122,7 @@ public abstract class ModelTestBase extends IntegrationTestBase
try
{
- validator.visitPartition(0);
+ validator.visit(0);
}
catch (Throwable t)
{
diff --git a/harry-integration/test/harry/model/QuerySelectorNegativeTest.java b/harry-integration/test/harry/model/QuerySelectorNegativeTest.java
index 928e4e9..92498dd 100644
--- a/harry-integration/test/harry/model/QuerySelectorNegativeTest.java
+++ b/harry-integration/test/harry/model/QuerySelectorNegativeTest.java
@@ -25,7 +25,6 @@ import java.util.Map;
import java.util.Random;
import java.util.function.Supplier;
-import harry.operations.Query;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -40,9 +39,10 @@ import harry.corruptor.HideValueCorruptor;
import harry.corruptor.QueryResponseCorruptor;
import harry.corruptor.ShowValueCorruptor;
import harry.ddl.SchemaGenerators;
-import harry.visitors.MutatingPartitionVisitor;
+import harry.visitors.MutatingVisitor;
import harry.visitors.MutatingRowVisitor;
-import harry.visitors.PartitionVisitor;
+import harry.visitors.Visitor;
+import harry.operations.Query;
import harry.operations.QueryGenerator;
import static harry.corruptor.QueryResponseCorruptor.SimpleQueryResponseCorruptor;
@@ -111,7 +111,7 @@ public class QuerySelectorNegativeTest extends IntegrationTestBase
System.out.println(run.schemaSpec.compile().cql());
OpSelectors.MonotonicClock clock = run.clock;
- PartitionVisitor partitionVisitor = new MutatingPartitionVisitor(run, MutatingRowVisitor::new);
+ Visitor visitor = new MutatingVisitor(run, MutatingRowVisitor::new);
Model model = new QuiescentChecker(run);
QueryResponseCorruptor corruptor = this.corruptorFactory.create(run);
@@ -119,7 +119,7 @@ public class QuerySelectorNegativeTest extends IntegrationTestBase
for (int i = 0; i < CYCLES; i++)
{
long lts = clock.nextLts();
- partitionVisitor.visitPartition(lts);
+ visitor.visit(lts);
}
while (true)
diff --git a/harry-integration/test/harry/model/QuerySelectorTest.java b/harry-integration/test/harry/model/QuerySelectorTest.java
index 3436809..6d8dc5c 100644
--- a/harry-integration/test/harry/model/QuerySelectorTest.java
+++ b/harry-integration/test/harry/model/QuerySelectorTest.java
@@ -31,9 +31,9 @@ import harry.ddl.SchemaGenerators;
import harry.ddl.SchemaSpec;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
-import harry.visitors.MutatingPartitionVisitor;
+import harry.visitors.MutatingVisitor;
import harry.visitors.MutatingRowVisitor;
-import harry.visitors.PartitionVisitor;
+import harry.visitors.Visitor;
import harry.operations.Query;
import harry.operations.QueryGenerator;
@@ -72,12 +72,12 @@ public class QuerySelectorTest extends IntegrationTestBase
run.sut.schemaChange(run.schemaSpec.compile().cql());
OpSelectors.MonotonicClock clock = run.clock;
- PartitionVisitor partitionVisitor = new MutatingPartitionVisitor(run, MutatingRowVisitor::new);
+ Visitor visitor = new MutatingVisitor(run, MutatingRowVisitor::new);
for (int i = 0; i < CYCLES; i++)
{
long lts = clock.nextLts();
- partitionVisitor.visitPartition(lts);
+ visitor.visit(lts);
}
QueryGenerator.TypedQueryGenerator querySelector = new QueryGenerator.TypedQueryGenerator(run);
@@ -146,12 +146,12 @@ public class QuerySelectorTest extends IntegrationTestBase
Run run = config.createRun();
run.sut.schemaChange(run.schemaSpec.compile().cql());
OpSelectors.MonotonicClock clock = run.clock;
- PartitionVisitor partitionVisitor = new MutatingPartitionVisitor(run, MutatingRowVisitor::new);
+ Visitor visitor = new MutatingVisitor(run, MutatingRowVisitor::new);
for (int i = 0; i < CYCLES; i++)
{
long lts = clock.nextLts();
- partitionVisitor.visitPartition(lts);
+ visitor.visit(lts);
}
QueryGenerator.TypedQueryGenerator querySelector = new QueryGenerator.TypedQueryGenerator(run);
diff --git a/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java b/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
index 55fb00b..945dce2 100644
--- a/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
+++ b/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
@@ -29,16 +29,16 @@ import harry.corruptor.HideValueCorruptor;
import harry.corruptor.QueryResponseCorruptor;
import harry.corruptor.QueryResponseCorruptor.SimpleQueryResponseCorruptor;
import harry.ddl.SchemaSpec;
-import harry.visitors.PartitionVisitor;
+import harry.visitors.Visitor;
import harry.operations.Query;
-import harry.visitors.SinglePartitionValidator;
+import harry.visitors.SingleValidator;
public class QuiescentCheckerIntegrationTest extends ModelTestBase
{
@Override
- protected PartitionVisitor validator(Run run)
+ protected Visitor validator(Run run)
{
- return new SinglePartitionValidator(100, run, modelConfiguration());
+ return new SingleValidator(100, run, modelConfiguration());
}
@Test
diff --git a/harry-integration/test/harry/model/TestEveryClustering.java b/harry-integration/test/harry/model/TestEveryClustering.java
deleted file mode 100644
index f3a2ba8..0000000
--- a/harry-integration/test/harry/model/TestEveryClustering.java
+++ /dev/null
@@ -1,83 +0,0 @@
-package harry.model;
-
-import harry.core.Configuration;
-import harry.core.Run;
-import harry.ddl.SchemaGenerators;
-import harry.ddl.SchemaSpec;
-import harry.generators.distribution.Distribution;
-import harry.operations.CompiledStatement;
-import harry.operations.Query;
-import harry.operations.Relation;
-import harry.visitors.LoggingPartitionVisitor;
-import harry.visitors.MutatingRowVisitor;
-import harry.visitors.PartitionVisitor;
-import org.apache.cassandra.distributed.api.IInvokableInstance;
-import org.junit.Test;
-
-import java.util.HashSet;
-import java.util.Set;
-import java.util.function.Supplier;
-
-public class TestEveryClustering extends IntegrationTestBase
-{
- int CYCLES = 1000;
-
- @Test
- public void basicQuerySelectorTest()
- {
- Supplier<SchemaSpec> schemaGen = SchemaGenerators.progression(SchemaGenerators.DEFAULT_SWITCH_AFTER);
- for (int cnt = 0; cnt < Integer.MAX_VALUE; cnt++)
- {
- beforeEach();
- SchemaSpec schemaSpec = schemaGen.get();
-
- System.out.println(schemaSpec.compile().cql());
- int partitionSize = 1000;
-
- Configuration config = sharedConfiguration(cnt, schemaSpec)
- .setPartitionDescriptorSelector(new Configuration.DefaultPDSelectorConfiguration(1, partitionSize))
- .setClusteringDescriptorSelector(sharedCDSelectorConfiguration()
- .setNumberOfModificationsDistribution(() -> new Distribution.ConstantDistribution(1L))
- .setRowsPerModificationDistribution(() -> new Distribution.ConstantDistribution(1L))
- .setMaxPartitionSize(250)
- .build())
- .build();
-
- Run run = config.createRun();
- run.sut.schemaChange(run.schemaSpec.compile().cql());
- OpSelectors.MonotonicClock clock = run.clock;
-
- Set<Long> visitedCds = new HashSet<>();
- PartitionVisitor partitionVisitor = new LoggingPartitionVisitor(run, (r) -> {
- return new MutatingRowVisitor(r) {
- public CompiledStatement perform(OpSelectors.OperationKind op, long lts, long pd, long cd, long opId)
- {
- visitedCds.add(cd);
- return super.perform(op, lts, pd, cd, opId);
- }
- };
- });
- sut.cluster().stream().forEach((IInvokableInstance node) -> node.nodetool("disableautocompaction"));
- for (int i = 0; i < CYCLES; i++)
- {
- long lts = clock.nextLts();
- partitionVisitor.visitPartition(lts);
-
- if (i > 0 && i % 250 == 0)
- sut.cluster().stream().forEach((IInvokableInstance node) -> node.nodetool("flush", schemaSpec.keyspace, schemaSpec.table));
- }
-
- for (Long cd : visitedCds)
- {
- Query query = new Query.SingleClusteringQuery(Query.QueryKind.SINGLE_CLUSTERING,
- run.pdSelector.pd(0),
- cd,
- false,
- Relation.eqRelations(run.schemaSpec.ckGenerator.slice(cd), run.schemaSpec.clusteringKeys),
- run.schemaSpec);
- Model model = new QuiescentChecker(run);
- model.validate(query);
- }
- }
- }
-}
diff --git a/harry-integration/test/resources/single_partition_test.yml b/harry-integration/test/resources/single_partition_test.yml
index 0ebe2aa..8ec4c4e 100644
--- a/harry-integration/test/resources/single_partition_test.yml
+++ b/harry-integration/test/resources/single_partition_test.yml
@@ -49,7 +49,7 @@ data_tracker:
runner:
sequential:
- partition_visitors: []
+ visitors: []
metric_reporter:
no_op: {}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra-harry] 01/05: Core improvements
Posted by if...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-harry.git
commit df40fa39ff1cee5165257a8334eee1ce78f829fe
Author: Alex Petrov <ol...@gmail.com>
AuthorDate: Mon Jul 12 17:04:37 2021 +0200
Core improvements
Major features:
* Implement updates
* Make sure we can advance RNGs from zero as well
* Fix a problem with predictable descriptor
Bugfixes:
* Fix column mask inconsistencies
* Fix a problem with partition key liveness info
Quality of life improvements:
* Get rid of driver dependency for query generation
* Get rid of guava dependency
* Add reusable config files
* Switch from streams to iterables
General improvements:
* Make unset and nil descriptors more distinct and harder to generate particularly for the smaller descriptors
* Fixed schema configurator to allow empty column sets
* Move workloads to a common dir
* Fixed schema configurator to output correct json
* No-op checker to execute with Quorum, not ALL
* Make tag for build unique
Patch by Alex Petrov for CASSANDRA-16262
---
harry-core/pom.xml | 9 +-
harry-core/src/harry/core/Configuration.java | 44 +++++-
harry-core/src/harry/core/VisibleForTesting.java | 5 +
harry-core/src/harry/corruptor/RowCorruptor.java | 2 +-
harry-core/src/harry/ddl/ColumnSpec.java | 10 +-
harry-core/src/harry/ddl/SchemaGenerators.java | 76 +++++----
harry-core/src/harry/ddl/SchemaSpec.java | 102 ++++++++++---
.../src/harry/generators/DataGenerators.java | 17 ++-
.../src/harry/generators/RandomGenerator.java | 3 +-
harry-core/src/harry/generators/RngUtils.java | 6 +-
harry-core/src/harry/generators/Surjections.java | 2 +-
.../harry/model/AlwaysSamePartitionSelector.java | 69 +++++++++
harry-core/src/harry/model/NoOpChecker.java | 3 +-
harry-core/src/harry/model/OpSelectors.java | 54 +++++--
harry-core/src/harry/model/SelectHelper.java | 114 ++++++++++----
.../model/clock/ApproximateMonotonicClock.java | 3 +-
harry-core/src/harry/model/clock/OffsetClock.java | 20 +++
harry-core/src/harry/model/sut/PrintlnSut.java | 18 +++
harry-core/src/harry/operations/DeleteHelper.java | 61 ++++----
harry-core/src/harry/operations/Relation.java | 80 ++--------
harry-core/src/harry/operations/WriteHelper.java | 170 +++++++++++----------
harry-core/src/harry/reconciler/Reconciler.java | 61 +++++---
.../harry/runner/CorruptingPartitionVisitor.java | 1 -
harry-core/src/harry/runner/DataTracker.java | 3 +
.../src/harry/runner/DefaultDataTracker.java | 2 +-
.../src/harry/runner/MutatingPartitionVisitor.java | 1 +
.../src/harry/runner/MutatingRowVisitor.java | 44 +++++-
harry-core/src/harry/runner/Operation.java | 20 ++-
harry-core/src/harry/runner/QueryGenerator.java | 2 +-
harry-core/src/harry/util/BitSet.java | 2 +-
harry-core/src/harry/util/TestRunner.java | 9 +-
harry-core/test/harry/model/OpSelectorsTest.java | 34 +++--
32 files changed, 690 insertions(+), 357 deletions(-)
diff --git a/harry-core/pom.xml b/harry-core/pom.xml
index fef070d..d313e7a 100755
--- a/harry-core/pom.xml
+++ b/harry-core/pom.xml
@@ -33,10 +33,11 @@
<name>Harry Core</name>
<dependencies>
- <dependency>
- <groupId>com.datastax.cassandra</groupId>
- <artifactId>cassandra-driver-core</artifactId>
- </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.25</version>
+ </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
diff --git a/harry-core/src/harry/core/Configuration.java b/harry-core/src/harry/core/Configuration.java
index f843e94..4a2a1e6 100644
--- a/harry-core/src/harry/core/Configuration.java
+++ b/harry-core/src/harry/core/Configuration.java
@@ -38,10 +38,13 @@ import harry.ddl.SchemaGenerators;
import harry.ddl.SchemaSpec;
import harry.generators.Surjections;
import harry.generators.distribution.Distribution;
+import harry.model.AlwaysSamePartitionSelector;
import harry.model.Model;
import harry.model.OpSelectors;
import harry.model.QuiescentChecker;
import harry.model.clock.ApproximateMonotonicClock;
+import harry.model.clock.OffsetClock;
+import harry.model.sut.PrintlnSut;
import harry.model.sut.SystemUnderTest;
import harry.runner.AllPartitionsValidator;
import harry.runner.CorruptingPartitionVisitor;
@@ -92,7 +95,10 @@ public class Configuration
mapper.registerSubtypes(CorruptingPartitionVisitorConfiguration.class);
mapper.registerSubtypes(RecentPartitionsValidatorConfiguration.class);
mapper.registerSubtypes(FixedSchemaProviderConfiguration.class);
-
+ mapper.registerSubtypes(AlwaysSamePartitionSelector.AlwaysSamePartitionSelectorConfiguration.class);
+ mapper.registerSubtypes(OffsetClock.OffsetClockConfiguration.class);
+ mapper.registerSubtypes(PrintlnSut.PrintlnSutConfiguration.class);
+ mapper.registerSubtypes(NoOpDataTrackerConfiguration.class);
mapper.registerSubtypes(NoOpMetricReporterConfiguration.class);
}
@@ -409,7 +415,7 @@ public class Configuration
}
- @JsonTypeName("no_op_tracker")
+ @JsonTypeName("no_op")
public static class NoOpDataTrackerConfiguration implements DataTrackerConfiguration
{
@JsonCreator
@@ -615,7 +621,7 @@ public class Configuration
}
}
- @JsonTypeName("no_op_checker")
+ @JsonTypeName("no_op")
public static class NoOpCheckerConfig implements ModelConfiguration
{
@JsonCreator
@@ -694,7 +700,7 @@ public class Configuration
private Map<OpSelectors.OperationKind, Integer> operation_kind_weights = new OperationKindSelectorBuilder()
.addWeight(OpSelectors.OperationKind.DELETE_ROW, 1)
.addWeight(OpSelectors.OperationKind.DELETE_COLUMN, 1)
- .addWeight(OpSelectors.OperationKind.WRITE, 98)
+ .addWeight(OpSelectors.OperationKind.INSERT, 98)
.build();
private Map<OpSelectors.OperationKind, long[]> column_mask_bitsets;
private int[] fractions;
@@ -1059,6 +1065,12 @@ public class Configuration
@JsonTypeName("fixed")
public static class FixedSchemaProviderConfiguration implements SchemaProviderConfiguration
{
+ public final String keyspace;
+ public final String table;
+ public final Map<String, String> partition_keys;
+ public final Map<String, String> clustering_keys;
+ public final Map<String, String> regular_columns;
+ public final Map<String, String> static_keys;
private final SchemaSpec schemaSpec;
@JsonCreator
@@ -1069,10 +1081,28 @@ public class Configuration
@JsonProperty("regular_columns") Map<String, String> regulars,
@JsonProperty("static_columns") Map<String, String> statics)
{
- this.schemaSpec = SchemaGenerators.parse(keyspace, table,
- pks, cks, regulars, statics);
+ this(SchemaGenerators.parse(keyspace, table,
+ pks, cks, regulars, statics),
+ pks,
+ cks,
+ regulars,
+ statics);
}
+ public FixedSchemaProviderConfiguration(SchemaSpec schemaSpec,
+ Map<String, String> pks,
+ Map<String, String> cks,
+ Map<String, String> regulars,
+ Map<String, String> statics)
+ {
+ this.schemaSpec = schemaSpec;
+ this.keyspace = schemaSpec.keyspace;
+ this.table = schemaSpec.table;
+ this.partition_keys = pks;
+ this.clustering_keys = cks;
+ this.regular_columns = regulars;
+ this.static_keys = statics;
+ }
public SchemaSpec make(long seed)
{
return schemaSpec;
@@ -1084,7 +1114,7 @@ public class Configuration
{
}
- @JsonTypeName("default")
+ @JsonTypeName("no_op")
public static class NoOpMetricReporterConfiguration implements MetricReporterConfiguration
{
public MetricReporter make()
diff --git a/harry-core/src/harry/core/VisibleForTesting.java b/harry-core/src/harry/core/VisibleForTesting.java
new file mode 100644
index 0000000..efa712e
--- /dev/null
+++ b/harry-core/src/harry/core/VisibleForTesting.java
@@ -0,0 +1,5 @@
+package harry.core;
+
+public @interface VisibleForTesting {
+}
+
diff --git a/harry-core/src/harry/corruptor/RowCorruptor.java b/harry-core/src/harry/corruptor/RowCorruptor.java
index 4c6b005..7b19cf1 100644
--- a/harry-core/src/harry/corruptor/RowCorruptor.java
+++ b/harry-core/src/harry/corruptor/RowCorruptor.java
@@ -29,7 +29,7 @@ import harry.operations.CompiledStatement;
public interface RowCorruptor
{
- Logger logger = LoggerFactory.getLogger(QueryResponseCorruptor.class);
+ final Logger logger = LoggerFactory.getLogger(QueryResponseCorruptor.class);
boolean canCorrupt(ResultSetRow row);
diff --git a/harry-core/src/harry/ddl/ColumnSpec.java b/harry-core/src/harry/ddl/ColumnSpec.java
index 94e9881..6d652c4 100644
--- a/harry-core/src/harry/ddl/ColumnSpec.java
+++ b/harry-core/src/harry/ddl/ColumnSpec.java
@@ -18,20 +18,18 @@
package harry.ddl;
+import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.UUID;
-import com.google.common.collect.ImmutableList;
-
import harry.generators.Bijections;
import harry.generators.StringBijection;
-import static harry.generators.StringBijection.getByte;
-
public class ColumnSpec<T>
{
public final String name;
@@ -318,7 +316,7 @@ public class ColumnSpec<T>
}
};
- public static final Collection<DataType<?>> DATA_TYPES = ImmutableList.of(
+ public static final Collection<DataType<?>> DATA_TYPES = Collections.unmodifiableList(Arrays.asList(
ColumnSpec.int8Type,
ColumnSpec.int16Type,
ColumnSpec.int32Type,
@@ -328,7 +326,7 @@ public class ColumnSpec<T>
ColumnSpec.doubleType,
ColumnSpec.asciiType,
ColumnSpec.uuidType,
- ColumnSpec.timestampType);
+ ColumnSpec.timestampType));
public static class ReversedType<T> extends DataType<T>
{
diff --git a/harry-core/src/harry/ddl/SchemaGenerators.java b/harry-core/src/harry/ddl/SchemaGenerators.java
index 47936e1..8508d44 100644
--- a/harry-core/src/harry/ddl/SchemaGenerators.java
+++ b/harry-core/src/harry/ddl/SchemaGenerators.java
@@ -21,16 +21,14 @@ package harry.ddl;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.function.Supplier;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
import harry.generators.Generator;
import harry.generators.Surjections;
@@ -43,32 +41,30 @@ public class SchemaGenerators
return new Builder(ks);
}
- public static final Collection<ColumnSpec.DataType<?>> clusteringKeyTypes;
public static final Map<String, ColumnSpec.DataType<?>> nameToTypeMap;
public static final Collection<ColumnSpec.DataType<?>> columnTypes;
+ public static final Collection<ColumnSpec.DataType<?>> partitionKeyTypes;
+ public static final Collection<ColumnSpec.DataType<?>> clusteringKeyTypes;
static
{
+ partitionKeyTypes = Collections.unmodifiableList(Arrays.asList(ColumnSpec.int64Type,
+ ColumnSpec.asciiType,
+ ColumnSpec.asciiType(4, 5),
+ ColumnSpec.asciiType(4, 10)));
+
+ columnTypes = Collections.unmodifiableList(Arrays.asList(
+// ColumnSpec.int8Type,
+// ColumnSpec.int16Type,
+// ColumnSpec.int32Type,
+ ColumnSpec.int64Type,
+ ColumnSpec.asciiType,
+ ColumnSpec.asciiType(4, 256),
+ ColumnSpec.asciiType(4, 512)));
+
- ImmutableList.Builder<ColumnSpec.DataType<?>> builder = ImmutableList.builder();
- builder.add(ColumnSpec.int8Type,
- ColumnSpec.int16Type,
- ColumnSpec.int32Type,
- ColumnSpec.int64Type,
- ColumnSpec.asciiType,
- ColumnSpec.asciiType(4, 256),
- ColumnSpec.asciiType(4, 512));
-
- columnTypes = builder.build();
- builder.add(ColumnSpec.int8Type,
- ColumnSpec.int16Type,
- ColumnSpec.int32Type,
- ColumnSpec.int64Type,
- ColumnSpec.asciiType);
- builder = ImmutableList.builder();
- builder.addAll(columnTypes);
-
- ImmutableMap.Builder<String, ColumnSpec.DataType<?>> mapBuilder = ImmutableMap.builder();
+ List<ColumnSpec.DataType<?>> builder = new ArrayList<>(columnTypes);
+ Map<String, ColumnSpec.DataType<?>> mapBuilder = new HashMap<>();
for (ColumnSpec.DataType<?> columnType : columnTypes)
{
@@ -82,8 +78,8 @@ public class SchemaGenerators
builder.add(ColumnSpec.floatType);
builder.add(ColumnSpec.doubleType);
- clusteringKeyTypes = builder.build();
- nameToTypeMap = mapBuilder.build();
+ clusteringKeyTypes = Collections.unmodifiableList(builder);
+ nameToTypeMap = Collections.unmodifiableMap(mapBuilder);
}
@SuppressWarnings("unchecked")
@@ -126,7 +122,7 @@ public class SchemaGenerators
public ColumnSpec<?> apply(ColumnSpec.DataType<?> type)
{
- return new ColumnSpec<>(prefix + (counter++),
+ return new ColumnSpec<>(String.format("%s%04d", prefix, counter++),
type,
kind);
}
@@ -143,7 +139,24 @@ public class SchemaGenerators
public ColumnSpec<?> apply(ColumnSpec.DataType<?> type)
{
- return ColumnSpec.ck(prefix + (counter++), type);
+ return ColumnSpec.ck(String.format("%s%04d", prefix, counter++), type);
+ }
+ });
+ }
+
+ @SuppressWarnings("unchecked")
+ public static Generator<ColumnSpec<?>> partitionColumnSpecGenerator(String prefix)
+ {
+ return fromValues(partitionKeyTypes)
+ .map(new Function<ColumnSpec.DataType<?>, ColumnSpec<?>>()
+ {
+ private int counter = 0;
+
+ public ColumnSpec<?> apply(ColumnSpec.DataType<?> type)
+ {
+
+ return ColumnSpec.pk(String.format("%s%04d", prefix, counter++),
+ type);
}
});
}
@@ -155,10 +168,10 @@ public class SchemaGenerators
private final String keyspace;
private final Supplier<String> tableNameSupplier;
- private Generator<ColumnSpec<?>> pkGenerator = columnSpecGenerator("pk", ColumnSpec.Kind.PARTITION_KEY);
+ private Generator<ColumnSpec<?>> pkGenerator = partitionColumnSpecGenerator("pk");
private Generator<ColumnSpec<?>> ckGenerator = clusteringColumnSpecGenerator("ck");
private Generator<ColumnSpec<?>> regularGenerator = columnSpecGenerator("regular", ColumnSpec.Kind.REGULAR);
- private Generator<ColumnSpec<?>> staticGenerator = columnSpecGenerator("regular", ColumnSpec.Kind.STATIC);
+ private Generator<ColumnSpec<?>> staticGenerator = columnSpecGenerator("static", ColumnSpec.Kind.STATIC);
private int minPks = 1;
private int maxPks = 1;
@@ -457,6 +470,9 @@ public class SchemaGenerators
public static List<ColumnSpec<?>> toColumns(Map<String, String> config, ColumnSpec.Kind kind, boolean allowReverse)
{
+ if (config == null)
+ return Collections.EMPTY_LIST;
+
List<ColumnSpec<?>> columns = new ArrayList<>(config.size());
for (Map.Entry<String, String> e : config.entrySet())
diff --git a/harry-core/src/harry/ddl/SchemaSpec.java b/harry-core/src/harry/ddl/SchemaSpec.java
index f07c106..eba87c6 100644
--- a/harry-core/src/harry/ddl/SchemaSpec.java
+++ b/harry-core/src/harry/ddl/SchemaSpec.java
@@ -18,14 +18,12 @@
package harry.ddl;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.function.Consumer;
-import java.util.stream.Stream;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Streams;
import harry.generators.DataGenerators;
import harry.operations.CompiledStatement;
@@ -87,22 +85,30 @@ public class SchemaSpec
this.keyspace = keyspace;
this.table = table;
this.isCompactStorage = isCompactStorage;
- this.partitionKeys = ImmutableList.copyOf(partitionKeys);
+
+ this.partitionKeys = Collections.unmodifiableList(new ArrayList<>(partitionKeys));
for (int i = 0; i < partitionKeys.size(); i++)
partitionKeys.get(i).setColumnIndex(i);
- this.clusteringKeys = ImmutableList.copyOf(clusteringKeys);
+ this.clusteringKeys = Collections.unmodifiableList(new ArrayList<>(clusteringKeys));
for (int i = 0; i < clusteringKeys.size(); i++)
clusteringKeys.get(i).setColumnIndex(i);
- this.staticColumns = ImmutableList.copyOf(staticColumns);
+ this.staticColumns = Collections.unmodifiableList(new ArrayList<>(staticColumns));
for (int i = 0; i < staticColumns.size(); i++)
staticColumns.get(i).setColumnIndex(i);
- this.regularColumns = ImmutableList.copyOf(regularColumns);
+ this.regularColumns = Collections.unmodifiableList(new ArrayList<>(regularColumns));
for (int i = 0; i < regularColumns.size(); i++)
regularColumns.get(i).setColumnIndex(i);
- this.allColumns = ImmutableList.copyOf(Iterables.concat(partitionKeys,
- clusteringKeys,
- staticColumns,
- regularColumns));
+
+ List<ColumnSpec<?>> all = new ArrayList<>();
+ for (ColumnSpec<?> columnSpec : concat(partitionKeys,
+ clusteringKeys,
+ staticColumns,
+ regularColumns))
+ {
+ all.add(columnSpec);
+ }
+ this.allColumns = Collections.unmodifiableList(all);
+
this.pkGenerator = DataGenerators.createKeyGenerator(partitionKeys);
this.ckGenerator = DataGenerators.createKeyGenerator(clusteringKeys);
@@ -122,7 +128,6 @@ public class SchemaSpec
}
// todo: bitset views?
-
public BitSet regularColumnsMask()
{
return this.regularColumnsMask;
@@ -254,13 +259,13 @@ public class SchemaSpec
sb.append(" PRIMARY KEY");
}
- Streams.concat(clusteringKeys.stream(),
- staticColumns.stream(),
- regularColumns.stream())
- .forEach((cd) -> {
- commaAppender.accept(sb);
- sb.append(cd.toCQL());
- });
+ for (ColumnSpec<?> cd : concat(clusteringKeys,
+ staticColumns,
+ regularColumns))
+ {
+ commaAppender.accept(sb);
+ sb.append(cd.toCQL());
+ }
if (clusteringKeys.size() > 0 || partitionKeys.size() > 1)
{
@@ -409,4 +414,59 @@ public class SchemaSpec
{
return Objects.hash(keyspace, table, partitionKeys, clusteringKeys, regularColumns);
}
+
+ public static <T> Iterable<T> concat(Iterable<T>... iterables)
+ {
+ assert iterables != null && iterables.length > 0;
+ if (iterables.length == 1)
+ return iterables[0];
+
+ return () -> {
+ return new Iterator<T>()
+ {
+ int idx;
+ Iterator<T> current;
+ boolean hasNext;
+
+ {
+ idx = 0;
+ prepareNext();
+ }
+
+ private void prepareNext()
+ {
+ if (current != null && current.hasNext())
+ {
+ hasNext = true;
+ return;
+ }
+
+ while (idx < iterables.length)
+ {
+ current = iterables[idx].iterator();
+ idx++;
+ if (current.hasNext())
+ {
+ hasNext = true;
+ return;
+ }
+ }
+
+ hasNext = false;
+ }
+
+ public boolean hasNext()
+ {
+ return hasNext;
+ }
+
+ public T next()
+ {
+ T next = current.next();
+ prepareNext();
+ return next;
+ }
+ };
+ };
+ }
}
diff --git a/harry-core/src/harry/generators/DataGenerators.java b/harry-core/src/harry/generators/DataGenerators.java
index 6878ae4..e87e7dc 100644
--- a/harry-core/src/harry/generators/DataGenerators.java
+++ b/harry-core/src/harry/generators/DataGenerators.java
@@ -22,15 +22,22 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import com.google.common.annotations.VisibleForTesting;
-
+import harry.core.VisibleForTesting;
import harry.ddl.ColumnSpec;
public class DataGenerators
{
- public static final Object UNSET_VALUE = new Object();
- public static long UNSET_DESCR = 0;
- public static long NIL_DESCR = -1;
+ public static final Object UNSET_VALUE = new Object() {
+ public String toString()
+ {
+ return "UNSET";
+ }
+ };
+
+ // There is still a slim chance that we're going to produce either of these values by chance, but we'll catch this
+ // during value generation
+ public static long UNSET_DESCR = Long.MAX_VALUE;
+ public static long NIL_DESCR = Long.MIN_VALUE;
public static Object[] inflateData(List<ColumnSpec<?>> columns, long[] descriptors)
{
diff --git a/harry-core/src/harry/generators/RandomGenerator.java b/harry-core/src/harry/generators/RandomGenerator.java
index 869f60e..a1ca125 100644
--- a/harry-core/src/harry/generators/RandomGenerator.java
+++ b/harry-core/src/harry/generators/RandomGenerator.java
@@ -18,8 +18,7 @@
package harry.generators;
-import com.google.common.annotations.VisibleForTesting;
-
+import harry.core.VisibleForTesting;
/**
* Random generator interface that offers:
diff --git a/harry-core/src/harry/generators/RngUtils.java b/harry-core/src/harry/generators/RngUtils.java
index 749cf7f..894204f 100644
--- a/harry-core/src/harry/generators/RngUtils.java
+++ b/harry-core/src/harry/generators/RngUtils.java
@@ -22,8 +22,12 @@ import java.util.function.LongSupplier;
public class RngUtils
{
+ private static final long CONSTANT = 0x2545F4914F6CDD1DL;
public static long next(long input)
{
+ if (input == 0)
+ return next(CONSTANT);
+
return xorshift64star(input);
}
@@ -32,7 +36,7 @@ public class RngUtils
input ^= input >> 12;
input ^= input << 25; // b
input ^= input >> 27; // c
- return input * 0x2545F4914F6CDD1DL;
+ return input * CONSTANT;
}
public static long[] next(long current, int n)
diff --git a/harry-core/src/harry/generators/Surjections.java b/harry-core/src/harry/generators/Surjections.java
index e5a937c..13f66ff 100644
--- a/harry-core/src/harry/generators/Surjections.java
+++ b/harry-core/src/harry/generators/Surjections.java
@@ -26,7 +26,7 @@ import java.util.function.Function;
import java.util.function.LongFunction;
import java.util.function.Supplier;
-import com.google.common.annotations.VisibleForTesting;
+import harry.core.VisibleForTesting;
public class Surjections
{
diff --git a/harry-core/src/harry/model/AlwaysSamePartitionSelector.java b/harry-core/src/harry/model/AlwaysSamePartitionSelector.java
new file mode 100644
index 0000000..43160fb
--- /dev/null
+++ b/harry-core/src/harry/model/AlwaysSamePartitionSelector.java
@@ -0,0 +1,69 @@
+package harry.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import harry.core.Configuration;
+
+/**
+ * A simple test-only descriptor selector that can used for testing things where you only need one partition
+ */
+public class AlwaysSamePartitionSelector extends OpSelectors.PdSelector
+{
+ private final long pd;
+
+ public AlwaysSamePartitionSelector(long pd)
+ {
+ this.pd = pd;
+ }
+
+ protected long pd(long lts)
+ {
+ return 0;
+ }
+
+ public long nextLts(long lts)
+ {
+ return lts + 1;
+ }
+
+ public long prevLts(long lts)
+ {
+ return lts - 1;
+ }
+
+ public long maxLtsFor(long pd)
+ {
+ return 1000;
+ }
+
+ public long minLtsAt(long position)
+ {
+ return 0;
+ }
+
+ public long minLtsFor(long pd)
+ {
+ return 0;
+ }
+
+ public long positionFor(long lts)
+ {
+ return 0;
+ }
+
+ @JsonTypeName("always_same")
+ public static class AlwaysSamePartitionSelectorConfiguration implements Configuration.PDSelectorConfiguration
+ {
+ private final long pd;
+
+ public AlwaysSamePartitionSelectorConfiguration(@JsonProperty("pd") long pd)
+ {
+ this.pd = pd;
+ }
+
+ public OpSelectors.PdSelector make(OpSelectors.Rng rng)
+ {
+ return new AlwaysSamePartitionSelector(pd);
+ }
+ }
+}
diff --git a/harry-core/src/harry/model/NoOpChecker.java b/harry-core/src/harry/model/NoOpChecker.java
index 4cf4606..a13b6ec 100644
--- a/harry-core/src/harry/model/NoOpChecker.java
+++ b/harry-core/src/harry/model/NoOpChecker.java
@@ -34,6 +34,7 @@ public class NoOpChecker implements Model
public void validate(Query query)
{
run.sut.execute(query.toSelectStatement(),
- SystemUnderTest.ConsistencyLevel.ALL);
+ // TODO: make it configurable
+ SystemUnderTest.ConsistencyLevel.QUORUM);
}
}
diff --git a/harry-core/src/harry/model/OpSelectors.java b/harry-core/src/harry/model/OpSelectors.java
index 3adbb95..3dd4a25 100644
--- a/harry-core/src/harry/model/OpSelectors.java
+++ b/harry-core/src/harry/model/OpSelectors.java
@@ -22,9 +22,8 @@ import java.util.EnumMap;
import java.util.List;
import java.util.Map;
-import com.google.common.annotations.VisibleForTesting;
-
import harry.core.Configuration;
+import harry.core.VisibleForTesting;
import harry.ddl.ColumnSpec;
import harry.ddl.SchemaSpec;
import harry.generators.Bytes;
@@ -34,6 +33,7 @@ import harry.generators.Surjections;
import harry.generators.distribution.Distribution;
import harry.util.BitSet;
+import static harry.generators.DataGenerators.NIL_DESCR;
import static harry.generators.DataGenerators.UNSET_DESCR;
/**
@@ -179,18 +179,20 @@ public interface OpSelectors
public long[] vds(long pd, long cd, long lts, long opId, SchemaSpec schema)
{
- return descriptors(pd, cd, lts, opId, schema.regularColumns, schema.regularColumnsMask(), schema.regularColumnsOffset);
+ BitSet setColumns = columnMask(pd, lts, opId);
+ return descriptors(pd, cd, lts, opId, schema.regularColumns, schema.regularColumnsMask(), setColumns, schema.regularColumnsOffset);
}
public long[] sds(long pd, long cd, long lts, long opId, SchemaSpec schema)
{
- return descriptors(pd, cd, lts, opId, schema.staticColumns, schema.staticColumnsMask(), schema.staticColumnsOffset);
+ BitSet setColumns = columnMask(pd, lts, opId);
+ return descriptors(pd, cd, lts, opId, schema.staticColumns, schema.staticColumnsMask(), setColumns, schema.staticColumnsOffset);
}
- public long[] descriptors(long pd, long cd, long lts, long opId, List<ColumnSpec<?>> columns, BitSet mask, int offset)
+ private long[] descriptors(long pd, long cd, long lts, long opId, List<ColumnSpec<?>> columns, BitSet mask, BitSet setColumns, int offset)
{
+ assert opId < opsPerModification(lts) * numberOfModifications(lts) : String.format("Operation id %d exceeds the maximum expected number of operations %d", opId, opsPerModification(lts) * numberOfModifications(lts));
long[] descriptors = new long[columns.size()];
- BitSet setColumns = columnMask(pd, cd, opId);
for (int i = 0; i < descriptors.length; i++)
{
@@ -199,6 +201,9 @@ public interface OpSelectors
{
ColumnSpec<?> spec = columns.get(i);
long vd = vd(pd, cd, lts, opId, col) & Bytes.bytePatternFor(spec.type.maxSize());
+ assert vd != UNSET_DESCR : "Ambiguous unset descriptor generated for the value";
+ assert vd != NIL_DESCR : "Ambiguous nil descriptor generated for the value";
+
descriptors[i] = vd;
}
else
@@ -387,13 +392,13 @@ public interface OpSelectors
switch (type)
{
+ case UPDATE_WITH_STATICS:
case DELETE_COLUMN_WITH_STATICS:
gen = (descriptor) -> {
long counter = 0;
while (counter <= 100)
{
BitSet bitSet = orig.inflate(descriptor);
-
if ((schema.regularColumns.isEmpty() || !bitSet.allUnset(schema.regularColumnsMask))
&& (schema.staticColumns.isEmpty() || !bitSet.allUnset(schema.staticColumnsMask)))
return bitSet;
@@ -404,6 +409,23 @@ public interface OpSelectors
throw new RuntimeException(String.format("Could not generate a value after %d attempts.", counter));
};
break;
+ // Can not have an UPDATE statement without anything to update
+ case UPDATE:
+ gen = descriptor -> {
+ long counter = 0;
+ while (counter <= 100)
+ {
+ BitSet bitSet = orig.inflate(descriptor);
+
+ if (!bitSet.allUnset(schema.regularColumnsMask))
+ return bitSet;
+
+ descriptor = RngUtils.next(descriptor);
+ counter++;
+ }
+ throw new RuntimeException(String.format("Could not generate a value after %d attempts.", counter));
+ };
+ break;
case DELETE_COLUMN:
gen = (descriptor) -> {
long counter = 0;
@@ -429,7 +451,7 @@ public interface OpSelectors
public ColumnSelectorBuilder forWrite(Surjections.Surjection<BitSet> gen)
{
- m.put(OperationKind.WRITE, gen);
+ m.put(OperationKind.INSERT, gen);
return this;
}
@@ -547,8 +569,8 @@ public interface OpSelectors
protected final static long BITSET_IDX_STREAM = 0x92eb607bef1L;
public static OperationSelector DEFAULT_OP_SELECTOR = OperationSelector.weighted(Surjections.weights(45, 45, 3, 2, 2, 1, 1, 1),
- OperationKind.WRITE,
- OperationKind.WRITE_WITH_STATICS,
+ OperationKind.INSERT,
+ OperationKind.INSERT_WITH_STATICS,
OperationKind.DELETE_ROW,
OperationKind.DELETE_COLUMN,
OperationKind.DELETE_COLUMN_WITH_STATICS,
@@ -649,7 +671,8 @@ public interface OpSelectors
public OperationKind operationType(long pd, long lts, long opId)
{
- return operationType(pd, lts, opId, partitionLevelOperationsMask(pd, lts));
+ OperationKind kind = operationType(pd, lts, opId, partitionLevelOperationsMask(pd, lts));
+ return kind;
}
// TODO: create this bitset once per lts
@@ -666,7 +689,8 @@ public interface OpSelectors
public OperationKind operationType(long pd, long lts, long opId, BitSet partitionLevelOperationsMask)
{
- return operationSelector.inflate(pd ^ lts ^ opId, partitionLevelOperationsMask.isSet((int) opId));
+ long descriptor = rng.randomNumber(pd ^ lts ^ opId, BITSET_IDX_STREAM);
+ return operationSelector.inflate(descriptor, partitionLevelOperationsMask.isSet((int) opId));
}
public BitSet columnMask(long pd, long lts, long opId)
@@ -688,8 +712,10 @@ public interface OpSelectors
public enum OperationKind
{
- WRITE(false),
- WRITE_WITH_STATICS(true),
+ UPDATE(false),
+ INSERT(false),
+ UPDATE_WITH_STATICS(true),
+ INSERT_WITH_STATICS(true),
DELETE_PARTITION(true),
DELETE_ROW(false),
DELETE_COLUMN(false),
diff --git a/harry-core/src/harry/model/SelectHelper.java b/harry-core/src/harry/model/SelectHelper.java
index 70d1eb2..fc8f6f7 100644
--- a/harry-core/src/harry/model/SelectHelper.java
+++ b/harry-core/src/harry/model/SelectHelper.java
@@ -19,13 +19,9 @@
package harry.model;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import com.datastax.driver.core.querybuilder.Ordering;
-import com.datastax.driver.core.querybuilder.QueryBuilder;
-import com.datastax.driver.core.querybuilder.Select;
import harry.data.ResultSetRow;
import harry.ddl.ColumnSpec;
import harry.ddl.SchemaSpec;
@@ -48,67 +44,119 @@ public class SelectHelper
*/
public static CompiledStatement select(SchemaSpec schema, long pd, List<Relation> relations, boolean reverse, boolean includeWriteTime)
{
- Select.Selection select = QueryBuilder.select();
- for (ColumnSpec<?> column : schema.allColumns)
- select.column(column.name);
+ StringBuilder b = new StringBuilder();
+ b.append("SELECT ");
+
+ for (int i = 0; i < schema.allColumns.size(); i++)
+ {
+ ColumnSpec<?> spec = schema.allColumns.get(i);
+ if (i > 0)
+ b.append(", ");
+ b.append(spec.name);
+ }
if (includeWriteTime)
{
for (ColumnSpec<?> column : schema.staticColumns)
- select.writeTime(column.name);
+ b.append(", ")
+ .append("writetime(")
+ .append(column.name)
+ .append(")");
for (ColumnSpec<?> column : schema.regularColumns)
- select.writeTime(column.name);
+ b.append(", ")
+ .append("writetime(")
+ .append(column.name)
+ .append(")");
}
- Select.Where where = select.from(schema.keyspace, schema.table).where();
- List<Object> bindings = new ArrayList<>();
+ b.append(" FROM ")
+ .append(schema.keyspace)
+ .append(".")
+ .append(schema.table)
+ .append(" WHERE ");
- addRelations(schema, where, bindings, pd, relations);
- addOrderBy(schema, where, reverse);
+ List<Object> bindings = new ArrayList<>();
+ schema.inflateRelations(pd,
+ relations,
+ new SchemaSpec.AddRelationCallback()
+ {
+ boolean isFirst = true;
+ public void accept(ColumnSpec<?> spec, Relation.RelationKind kind, Object value)
+ {
+ if (isFirst)
+ isFirst = false;
+ else
+ b.append(" AND ");
+ b.append(kind.getClause(spec));
+ bindings.add(value);
+ }
+ });
+ addOrderBy(schema, b, reverse);
+ b.append(";");
Object[] bindingsArr = bindings.toArray(new Object[bindings.size()]);
- return new CompiledStatement(where.toString(), bindingsArr);
+ return new CompiledStatement(b.toString(), bindingsArr);
}
public static CompiledStatement count(SchemaSpec schema, long pd)
{
- Select.Selection select = QueryBuilder.select();
- select.countAll();
-
- Select.Where where = select.from(schema.keyspace, schema.table).where();
- List<Object> bindings = new ArrayList<>(schema.partitionKeys.size());
+ StringBuilder b = new StringBuilder();
+ b.append("SELECT count(*) ");
- addRelations(schema, where, bindings, pd, Collections.emptyList());
+ b.append(" FROM ")
+ .append(schema.keyspace)
+ .append(".")
+ .append(schema.table)
+ .append(" WHERE ");
- Object[] bindingsArr = bindings.toArray(new Object[bindings.size()]);
- return new CompiledStatement(where.toString(), bindingsArr);
- }
+ List<Object> bindings = new ArrayList<>(schema.partitionKeys.size());
- private static void addRelations(SchemaSpec schema, Select.Where where, List<Object> bindings, long pd, List<Relation> relations)
- {
schema.inflateRelations(pd,
- relations,
- (spec, kind, value) -> {
- where.and(kind.getClause(spec));
- bindings.add(value);
+ Collections.emptyList(),
+ new SchemaSpec.AddRelationCallback()
+ {
+ boolean isFirst = true;
+ public void accept(ColumnSpec<?> spec, Relation.RelationKind kind, Object value)
+ {
+ if (isFirst)
+ isFirst = false;
+ else
+ b.append(" AND ");
+ b.append(kind.getClause(spec));
+ bindings.add(value);
+ }
});
+
+ Object[] bindingsArr = bindings.toArray(new Object[bindings.size()]);
+ return new CompiledStatement(b.toString(), bindingsArr);
}
- private static void addOrderBy(SchemaSpec schema, Select.Where whereClause, boolean reverse)
+ private static void addOrderBy(SchemaSpec schema, StringBuilder b, boolean reverse)
{
if (reverse && schema.clusteringKeys.size() > 0)
{
- Ordering[] ordering = new Ordering[schema.clusteringKeys.size()];
+ b.append(" ORDER BY ");
for (int i = 0; i < schema.clusteringKeys.size(); i++)
{
ColumnSpec<?> c = schema.clusteringKeys.get(i);
- ordering[i] = c.isReversed() ? QueryBuilder.asc(c.name) : QueryBuilder.desc(c.name);
+ if (i > 0)
+ b.append(", ");
+ b.append(c.isReversed() ? asc(c.name) : desc(c.name));
}
- whereClause.orderBy(ordering);
}
}
+ public static String asc(String name)
+ {
+ return name + " ASC";
+ }
+
+ public static String desc(String name)
+ {
+ return name + " DESC";
+ }
+
public static ResultSetRow resultSetToRow(SchemaSpec schema, OpSelectors.MonotonicClock clock, Object[] result)
{
Object[] partitionKey = new Object[schema.partitionKeys.size()];
diff --git a/harry-core/src/harry/model/clock/ApproximateMonotonicClock.java b/harry-core/src/harry/model/clock/ApproximateMonotonicClock.java
index 59dd47e..1a64500 100644
--- a/harry-core/src/harry/model/clock/ApproximateMonotonicClock.java
+++ b/harry-core/src/harry/model/clock/ApproximateMonotonicClock.java
@@ -25,9 +25,8 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicLongArray;
import java.util.concurrent.locks.LockSupport;
-import com.google.common.annotations.VisibleForTesting;
-
import harry.core.Configuration;
+import harry.core.VisibleForTesting;
import harry.model.OpSelectors;
/**
diff --git a/harry-core/src/harry/model/clock/OffsetClock.java b/harry-core/src/harry/model/clock/OffsetClock.java
index 9f40a64..8c25394 100644
--- a/harry-core/src/harry/model/clock/OffsetClock.java
+++ b/harry-core/src/harry/model/clock/OffsetClock.java
@@ -20,6 +20,9 @@ package harry.model.clock;
import java.util.concurrent.atomic.AtomicLong;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
import harry.core.Configuration;
import harry.model.OpSelectors;
@@ -58,4 +61,21 @@ public class OffsetClock implements OpSelectors.MonotonicClock
{
throw new RuntimeException("not implemented");
}
+
+ @JsonTypeName("offset")
+ public static class OffsetClockConfiguration implements Configuration.ClockConfiguration
+ {
+ public final long offset;
+
+ @JsonCreator
+ public OffsetClockConfiguration(@JsonProperty("offset") int offset)
+ {
+ this.offset = offset;
+ }
+
+ public OpSelectors.MonotonicClock make()
+ {
+ return new OffsetClock(offset);
+ }
+ }
}
diff --git a/harry-core/src/harry/model/sut/PrintlnSut.java b/harry-core/src/harry/model/sut/PrintlnSut.java
index ad14b0b..f1b310e 100644
--- a/harry-core/src/harry/model/sut/PrintlnSut.java
+++ b/harry-core/src/harry/model/sut/PrintlnSut.java
@@ -21,6 +21,10 @@ package harry.model.sut;
import java.util.Arrays;
import java.util.concurrent.CompletableFuture;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import harry.core.Configuration;
+
public class PrintlnSut implements SystemUnderTest
{
public boolean isShutdown()
@@ -46,4 +50,18 @@ public class PrintlnSut implements SystemUnderTest
return CompletableFuture.supplyAsync(() -> execute(statement, cl, bindings),
Runnable::run);
}
+
+ @JsonTypeName("println")
+ public static class PrintlnSutConfiguration implements Configuration.SutConfiguration
+ {
+ @JsonCreator
+ public PrintlnSutConfiguration()
+ {
+
+ }
+ public SystemUnderTest make()
+ {
+ return new PrintlnSut();
+ }
+ }
}
diff --git a/harry-core/src/harry/operations/DeleteHelper.java b/harry-core/src/harry/operations/DeleteHelper.java
index 6f04bad..f1b6983 100644
--- a/harry-core/src/harry/operations/DeleteHelper.java
+++ b/harry-core/src/harry/operations/DeleteHelper.java
@@ -23,11 +23,8 @@ import java.util.Collections;
import java.util.List;
import java.util.function.IntConsumer;
-import com.datastax.driver.core.querybuilder.Delete;
-import com.datastax.driver.core.querybuilder.QueryBuilder;
import harry.ddl.ColumnSpec;
import harry.ddl.SchemaSpec;
-import harry.runner.LoggingPartitionVisitor;
import harry.util.BitSet;
public class DeleteHelper
@@ -130,40 +127,50 @@ public class DeleteHelper
BitSet mask,
long ts)
{
- Delete delete;
- if (columnsToDelete == null)
- delete = QueryBuilder.delete().from(schema.keyspace, schema.table);
- else
+ StringBuilder b = new StringBuilder();
+ b.append("DELETE ");
+ if (columnsToDelete != null)
{
assert mask != null;
assert relations == null || relations.stream().allMatch((r) -> r.kind == Relation.RelationKind.EQ);
- delete = QueryBuilder.delete(columnNames(schema.allColumns, columnsToDelete, mask))
- .from(schema.keyspace, schema.table);
+ String[] names = columnNames(schema.allColumns, columnsToDelete, mask);
+ for (int i = 0; i < names.length; i++)
+ {
+ if (i > 0)
+ b.append(", ");
+ b.append(names[i]);
+ }
+ b.append(" ");
}
+ b.append("FROM ")
+ .append(schema.keyspace).append(".").append(schema.table)
+ .append(" USING TIMESTAMP ")
+ .append(ts)
+ .append(" WHERE ");
- Delete.Where where = delete.where();
List<Object> bindings = new ArrayList<>();
- addRelations(schema, where, bindings, pd, relations);
- delete.using(QueryBuilder.timestamp(ts));
- delete.setForceNoValues(true);
- Object[] bindingsArr = bindings.toArray(new Object[bindings.size()]);
- String compiled = delete.getQueryString();
- if (compiled.contains("built query (could not generate with default codec registry:"))
- throw new IllegalArgumentException(String.format("Could not generate the query: %s. Bindings: (%s)",
- delete,
- CompiledStatement.bindingsToString(bindingsArr)));
- return new CompiledStatement(compiled, bindingsArr);
- }
-
- private static void addRelations(SchemaSpec schema, Delete.Where where, List<Object> bindings, long pd, List<Relation> relations)
- {
schema.inflateRelations(pd,
relations,
- (spec, kind, value) -> {
- where.and(kind.getClause(spec));
- bindings.add(value);
+ new SchemaSpec.AddRelationCallback()
+ {
+ boolean isFirst = true;
+ public void accept(ColumnSpec<?> spec, Relation.RelationKind kind, Object value)
+ {
+ if (isFirst)
+ isFirst = false;
+ else
+ b.append(" AND ");
+ b.append(kind.getClause(spec));
+ bindings.add(value);
+ }
});
+
+ b.append(";");
+
+ Object[] bindingsArr = bindings.toArray(new Object[bindings.size()]);
+
+ return new CompiledStatement(b.toString(), bindingsArr);
}
private static String[] columnNames(List<ColumnSpec<?>> columns, BitSet selectedColumns, BitSet mask)
diff --git a/harry-core/src/harry/operations/Relation.java b/harry-core/src/harry/operations/Relation.java
index 19db0b4..87487d9 100644
--- a/harry-core/src/harry/operations/Relation.java
+++ b/harry-core/src/harry/operations/Relation.java
@@ -21,16 +21,8 @@ package harry.operations;
import java.util.ArrayList;
import java.util.List;
-import com.datastax.driver.core.querybuilder.Clause;
import harry.ddl.ColumnSpec;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.bindMarker;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.eq;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.gt;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.gte;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.lt;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.lte;
-
public class Relation
{
public final RelationKind kind;
@@ -62,9 +54,9 @@ public class Relation
return columnSpec.name;
}
- public Clause toClause()
+ public String toClause()
{
- return kind.getClause(column(), bindMarker());
+ return kind.getClause(column());
}
public String toString()
@@ -101,7 +93,7 @@ public class Relation
public static void addRelation(long[] key, List<ColumnSpec<?>> columnSpecs, List<Relation> relations, RelationKind kind)
{
assert key.length == columnSpecs.size() :
- String.format("Key size (%d) should equal to column spec size (%d)", key.length, columnSpecs.size());
+ String.format("Key size (%d) should equal to column spec size (%d). Specs: %s", key.length, columnSpecs.size(), columnSpecs);
for (int i = 0; i < key.length; i++)
{
ColumnSpec<?> spec = columnSpecs.get(i);
@@ -113,17 +105,6 @@ public class Relation
{
LT
{
- @Override
- public Clause getClause(String name, Object obj)
- {
- return lt(name, obj);
- }
-
- public Clause getClause(List<String> name, List<Object> obj)
- {
- return lt(name, obj);
- }
-
public boolean isNegatable()
{
return true;
@@ -156,17 +137,6 @@ public class Relation
},
GT
{
- @Override
- public Clause getClause(String name, Object obj)
- {
- return gt(name, obj);
- }
-
- public Clause getClause(List<String> name, List<Object> obj)
- {
- return gt(name, obj);
- }
-
public boolean isNegatable()
{
return true;
@@ -199,17 +169,6 @@ public class Relation
},
LTE
{
- @Override
- public Clause getClause(String name, Object obj)
- {
- return lte(name, obj);
- }
-
- public Clause getClause(List<String> name, List<Object> obj)
- {
- return lt(name, obj);
- }
-
public boolean isNegatable()
{
return true;
@@ -242,17 +201,6 @@ public class Relation
},
GTE
{
- @Override
- public Clause getClause(String name, Object obj)
- {
- return gte(name, obj);
- }
-
- public Clause getClause(List<String> name, List<Object> obj)
- {
- return gte(name, obj);
- }
-
public boolean isNegatable()
{
return true;
@@ -285,17 +233,6 @@ public class Relation
},
EQ
{
- @Override
- public Clause getClause(String name, Object obj)
- {
- return eq(name, obj);
- }
-
- public Clause getClause(List<String> name, List<Object> obj)
- {
- return eq(name, obj);
- }
-
public boolean isNegatable()
{
return false;
@@ -329,14 +266,15 @@ public class Relation
public abstract boolean match(LongComparator comparator, long l, long r);
- public abstract Clause getClause(String name, Object obj);
-
- public Clause getClause(ColumnSpec<?> spec)
+ public String getClause(String name)
{
- return getClause(spec.name, bindMarker());
+ return String.format("%s %s ?", name, toString());
}
- public abstract Clause getClause(List<String> name, List<Object> obj);
+ public String getClause(ColumnSpec<?> spec)
+ {
+ return getClause(spec.name);
+ }
public abstract boolean isNegatable();
diff --git a/harry-core/src/harry/operations/WriteHelper.java b/harry-core/src/harry/operations/WriteHelper.java
index a0b7565..084f931 100644
--- a/harry-core/src/harry/operations/WriteHelper.java
+++ b/harry-core/src/harry/operations/WriteHelper.java
@@ -18,21 +18,13 @@
package harry.operations;
+import java.util.Arrays;
import java.util.List;
import harry.ddl.ColumnSpec;
import harry.ddl.SchemaSpec;
import harry.generators.DataGenerators;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.bindMarker;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.eq;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.in;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.insertInto;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.set;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.timestamp;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.truncate;
-import static com.datastax.driver.core.querybuilder.QueryBuilder.update;
-
public class WriteHelper
{
public static CompiledStatement inflateInsert(SchemaSpec schema,
@@ -48,117 +40,127 @@ public class WriteHelper
Object[] regularColumns = schema.inflateRegularColumns(vds);
Object[] bindings = new Object[schema.allColumns.size()];
- int bindingsCount = 0;
- com.datastax.driver.core.querybuilder.Insert insert = insertInto(schema.keyspace,
- schema.table);
- bindingsCount += addValue(insert, bindings, schema.partitionKeys, partitionKey, bindingsCount);
- bindingsCount += addValue(insert, bindings, schema.clusteringKeys, clusteringKey, bindingsCount);
+ StringBuilder b = new StringBuilder();
+ b.append("INSERT INTO ")
+ .append(schema.keyspace)
+ .append('.')
+ .append(schema.table)
+ .append(" (");
+
+ int bindingsCount = 0;
+ bindingsCount += appendStatements(b, bindings, schema.partitionKeys, partitionKey, bindingsCount, true, ",", "%s");
+ bindingsCount += appendStatements(b, bindings, schema.clusteringKeys, clusteringKey, bindingsCount, false, ",", "%s");
+ bindingsCount += appendStatements(b, bindings, schema.regularColumns, regularColumns, bindingsCount, false, ",", "%s");
if (staticColumns != null)
- bindingsCount += addValue(insert, bindings, schema.staticColumns, staticColumns, bindingsCount);
- bindingsCount += addValue(insert, bindings, schema.regularColumns, regularColumns, bindingsCount);
+ bindingsCount += appendStatements(b, bindings, schema.staticColumns, staticColumns, bindingsCount, false, ",", "%s");
- insert.using(timestamp(timestamp));
+ b.append(") VALUES (");
- // Some of the values were unset
- if (bindingsCount != bindings.length)
+ for (int i = 0; i < bindingsCount; i++)
{
- Object[] tmp = new Object[bindingsCount];
- System.arraycopy(bindings, 0, tmp, 0, bindingsCount);
- bindings = tmp;
+ if (i > 0)
+ b.append(", ");
+ b.append("?");
}
- return CompiledStatement.create(insert.toString(), bindings);
+ b.append(") USING TIMESTAMP ")
+ .append(timestamp);
+
+ return new CompiledStatement(b.toString(), adjustArraySize(bindings, bindingsCount));
}
- public static boolean allUnset(long[] descriptors)
+ public static Object[] adjustArraySize(Object[] bindings, int bindingsCount)
{
- for (long descriptor : descriptors)
+ if (bindingsCount != bindings.length)
{
- if (descriptor != DataGenerators.UNSET_DESCR)
- return false;
+ Object[] tmp = new Object[bindingsCount];
+ System.arraycopy(bindings, 0, tmp, 0, bindingsCount);
+ bindings = tmp;
}
- return true;
- }
- private static int addValue(com.datastax.driver.core.querybuilder.Insert insert,
- Object[] bindings,
- List<ColumnSpec<?>> columns,
- Object[] data,
- int bound)
- {
- assert data.length == columns.size();
-
- int bindingsCount = 0;
- for (int i = 0; i < data.length; i++)
- {
- if (data[i] == DataGenerators.UNSET_VALUE)
- continue;
-
- insert.value(columns.get(i).name, bindMarker());
- bindings[bound + bindingsCount] = data[i];
- bindingsCount++;
- }
-
- return bindingsCount;
+ return bindings;
}
public static CompiledStatement inflateUpdate(SchemaSpec schema,
long pd,
long cd,
long[] vds,
+ long[] sds,
long timestamp)
{
Object[] partitionKey = schema.inflatePartitionKey(pd);
Object[] clusteringKey = schema.inflateClusteringKey(cd);
+ Object[] staticColumns = sds == null ? null : schema.inflateStaticColumns(sds);
Object[] regularColumns = schema.inflateRegularColumns(vds);
Object[] bindings = new Object[schema.allColumns.size()];
- int bindingsCount = 0;
- com.datastax.driver.core.querybuilder.Update update = update(schema.keyspace,
- schema.table);
- bindingsCount += addWith(update, bindings, schema.regularColumns, regularColumns, bindingsCount);
- bindingsCount += addWhere(update, bindings, schema.partitionKeys, partitionKey, bindingsCount);
- bindingsCount += addWhere(update, bindings, schema.clusteringKeys, clusteringKey, bindingsCount);
+ StringBuilder b = new StringBuilder();
+ b.append("UPDATE ")
+ .append(schema.keyspace)
+ .append('.')
+ .append(schema.table)
+ .append(" USING TIMESTAMP ")
+ .append(timestamp)
+ .append(" SET ");
- update.using(timestamp(timestamp));
- // TODO: TTL
- // ttl.ifPresent(ts -> update.using(ttl(ts)));
+ int bindingsCount = 0;
+ bindingsCount += addSetStatements(b, bindings, schema.regularColumns, regularColumns, bindingsCount);
+ if (staticColumns != null)
+ bindingsCount += addSetStatements(b, bindings, schema.staticColumns, staticColumns, bindingsCount);
+
+ assert bindingsCount > 0 : "Can not have an UPDATE statement without any updates";
+ b.append(" WHERE ");
- return CompiledStatement.create(update.toString(), bindings);
+ bindingsCount += addWhereStatements(b, bindings, schema.partitionKeys, partitionKey, bindingsCount, true);
+ bindingsCount += addWhereStatements(b, bindings, schema.clusteringKeys, clusteringKey, bindingsCount, false);
+ b.append(";");
+ return new CompiledStatement(b.toString(), adjustArraySize(bindings, bindingsCount));
}
- private static int addWith(com.datastax.driver.core.querybuilder.Update update,
- Object[] bindings,
- List<ColumnSpec<?>> columns,
- Object[] data,
- int bound)
+ private static int addSetStatements(StringBuilder b,
+ Object[] bindings,
+ List<ColumnSpec<?>> columns,
+ Object[] values,
+ int bound)
{
- assert data.length == columns.size();
-
- for (int i = 0; i < data.length; i++)
- {
- update.with(set(columns.get(i).name, bindMarker()));
- bindings[bound + i] = data[i];
- }
-
- return data.length;
+ return appendStatements(b, bindings, columns, values, bound, bound == 0, ", ", "%s = ?");
}
- private static int addWhere(com.datastax.driver.core.querybuilder.Update update,
- Object[] bindings,
- List<ColumnSpec<?>> columns,
- Object[] data,
- int bound)
+ private static int addWhereStatements(StringBuilder b,
+ Object[] bindings,
+ List<ColumnSpec<?>> columns,
+ Object[] values,
+ int bound,
+ boolean firstStatement)
{
- assert data.length == columns.size();
+ return appendStatements(b, bindings, columns, values, bound, firstStatement, " AND ", "%s = ?");
+ }
- for (int i = 0; i < data.length; i++)
+ private static int appendStatements(StringBuilder b,
+ Object[] allBindings,
+ List<ColumnSpec<?>> columns,
+ Object[] values,
+ int bound,
+ boolean firstStatement,
+ String separator,
+ String nameFormatter)
+ {
+ int bindingsCount = 0;
+ for (int i = 0; i < values.length; i++)
{
- update.where().and(eq(columns.get(i).name, bindMarker()));
- bindings[bound + i] = data[i];
- }
+ Object value = values[i];
+ if (value == DataGenerators.UNSET_VALUE)
+ continue;
+
+ ColumnSpec<?> column = columns.get(i);
+ if (bindingsCount > 0 || !firstStatement)
+ b.append(separator);
- return data.length;
+ b.append(String.format(nameFormatter, column.name));
+ allBindings[bound + bindingsCount] = value;
+ bindingsCount++;
+ }
+ return bindingsCount;
}
}
\ No newline at end of file
diff --git a/harry-core/src/harry/reconciler/Reconciler.java b/harry-core/src/harry/reconciler/Reconciler.java
index ca4772c..a9709b3 100644
--- a/harry-core/src/harry/reconciler/Reconciler.java
+++ b/harry-core/src/harry/reconciler/Reconciler.java
@@ -125,8 +125,10 @@ public class Reconciler
hadPartitionDeletion = true;
break;
- case WRITE_WITH_STATICS:
- case WRITE:
+ case INSERT_WITH_STATICS:
+ case INSERT:
+ case UPDATE:
+ case UPDATE_WITH_STATICS:
if (debugCd != -1 && cd == debugCd)
logger.info("Writing {} ({}) at {}/{}", cd, opType, lts, opId);
writes.add(opId);
@@ -164,12 +166,14 @@ public class Reconciler
switch (opType)
{
- case WRITE_WITH_STATICS:
+ case INSERT_WITH_STATICS:
+ case UPDATE_WITH_STATICS:
// We could apply static columns during the first iteration, but it's more convenient
// to reconcile static-level deletions.
partitionState.writeStaticRow(descriptorSelector.sds(pd, cd, lts, opId, schema),
lts);
- case WRITE:
+ case INSERT:
+ case UPDATE:
if (!query.match(cd))
{
if (debugCd != -1 && cd == debugCd)
@@ -189,7 +193,8 @@ public class Reconciler
partitionState.write(cd,
descriptorSelector.vds(pd, cd, lts, opId, schema),
- lts);
+ lts,
+ opType == OpSelectors.OperationKind.INSERT || opType == OpSelectors.OperationKind.INSERT_WITH_STATICS);
break;
default:
throw new IllegalStateException();
@@ -206,7 +211,8 @@ public class Reconciler
switch (opType)
{
case DELETE_COLUMN_WITH_STATICS:
- partitionState.deleteStaticColumns(schema.staticColumnsOffset,
+ partitionState.deleteStaticColumns(lts,
+ schema.staticColumnsOffset,
descriptorSelector.columnMask(pd, lts, opId),
schema.staticColumnsMask());
case DELETE_COLUMN:
@@ -227,7 +233,8 @@ public class Reconciler
}
}
- partitionState.deleteRegularColumns(cd,
+ partitionState.deleteRegularColumns(lts,
+ cd,
schema.regularColumnsOffset,
descriptorSelector.columnMask(pd, lts, opId),
schema.regularColumnsMask());
@@ -270,14 +277,15 @@ public class Reconciler
long lts)
{
if (staticRow != null)
- staticRow = updateRowState(staticRow, schema.staticColumns, STATIC_CLUSTERING, staticVds, lts);
+ staticRow = updateRowState(staticRow, schema.staticColumns, STATIC_CLUSTERING, staticVds, lts, false);
}
private void write(long cd,
long[] vds,
- long lts)
+ long lts,
+ boolean writeParimaryKeyLiveness)
{
- rows.compute(cd, (cd_, current) -> updateRowState(current, schema.regularColumns, cd, vds, lts));
+ rows.compute(cd, (cd_, current) -> updateRowState(current, schema.regularColumns, cd, vds, lts, writeParimaryKeyLiveness));
}
private void delete(Ranges.Range range,
@@ -304,14 +312,19 @@ public class Reconciler
private void delete(long cd,
long lts)
{
- rows.remove(cd);
+ RowState state = rows.remove(cd);
+ if (state != null)
+ {
+ for (long v : state.lts)
+ assert lts >= v : String.format("Attempted to remove a row with a tombstone that has older timestamp (%d): %s", lts, state);
+ }
}
public boolean isEmpty()
{
return rows.isEmpty();
}
- private RowState updateRowState(RowState currentState, List<ColumnSpec<?>> columns, long cd, long[] vds, long lts)
+ private RowState updateRowState(RowState currentState, List<ColumnSpec<?>> columns, long cd, long[] vds, long lts, boolean writePrimaryKeyLiveness)
{
if (currentState == null)
{
@@ -359,24 +372,30 @@ public class Reconciler
}
}
+ if (writePrimaryKeyLiveness)
+ currentState.hasPrimaryKeyLivenessInfo = true;
+
return currentState;
}
- private void deleteRegularColumns(long cd, int columnOffset, BitSet columns, BitSet mask)
+ private void deleteRegularColumns(long lts, long cd, int columnOffset, BitSet columns, BitSet mask)
{
- deleteColumns(rows.get(cd), columnOffset, columns, mask);
+ deleteColumns(lts, rows.get(cd), columnOffset, columns, mask);
}
- private void deleteStaticColumns(int columnOffset, BitSet columns, BitSet mask)
+ private void deleteStaticColumns(long lts, int columnOffset, BitSet columns, BitSet mask)
{
- deleteColumns(staticRow, columnOffset, columns, mask);
+ deleteColumns(lts, staticRow, columnOffset, columns, mask);
}
- private void deleteColumns(RowState state, int columnOffset, BitSet columns, BitSet mask)
+ private void deleteColumns(long lts, RowState state, int columnOffset, BitSet columns, BitSet mask)
{
if (state == null)
return;
+ //TODO: optimise by iterating over the columns that were removed by this deletion
+ //TODO: optimise final decision to fully remove the column by counting a number of set/unset columns
+ boolean allNil = true;
for (int i = 0; i < state.vds.length; i++)
{
if (columns.isSet(columnOffset + i, mask))
@@ -384,7 +403,14 @@ public class Reconciler
state.vds[i] = NIL_DESCR;
state.lts[i] = NO_TIMESTAMP;
}
+ else if (state.vds[i] != NIL_DESCR)
+ {
+ allNil = false;
+ }
}
+
+ if (state.cd != STATIC_CLUSTERING && allNil & !state.hasPrimaryKeyLivenessInfo)
+ delete(state.cd, lts);
}
private void deletePartition(long lts)
@@ -449,6 +475,7 @@ public class Reconciler
public static class RowState
{
+ public boolean hasPrimaryKeyLivenessInfo = false;
public final long cd;
public final long[] vds;
public final long[] lts;
diff --git a/harry-core/src/harry/runner/CorruptingPartitionVisitor.java b/harry-core/src/harry/runner/CorruptingPartitionVisitor.java
index ca5bb93..d079cf3 100644
--- a/harry-core/src/harry/runner/CorruptingPartitionVisitor.java
+++ b/harry-core/src/harry/runner/CorruptingPartitionVisitor.java
@@ -15,7 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package harry.runner;
import java.util.Random;
diff --git a/harry-core/src/harry/runner/DataTracker.java b/harry-core/src/harry/runner/DataTracker.java
index d5825fb..dab6fc9 100644
--- a/harry-core/src/harry/runner/DataTracker.java
+++ b/harry-core/src/harry/runner/DataTracker.java
@@ -18,6 +18,8 @@
package harry.runner;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonTypeName;
import harry.core.Configuration;
public interface DataTracker
@@ -35,6 +37,7 @@ public interface DataTracker
}
public static DataTracker NO_OP = new NoOpDataTracker();
+
class NoOpDataTracker implements DataTracker
{
private NoOpDataTracker() {}
diff --git a/harry-core/src/harry/runner/DefaultDataTracker.java b/harry-core/src/harry/runner/DefaultDataTracker.java
index 1b55482..7b1412a 100644
--- a/harry-core/src/harry/runner/DefaultDataTracker.java
+++ b/harry-core/src/harry/runner/DefaultDataTracker.java
@@ -23,11 +23,11 @@ import java.util.List;
import java.util.concurrent.PriorityBlockingQueue;
import java.util.concurrent.atomic.AtomicLong;
-import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import harry.core.Configuration;
+import harry.core.VisibleForTesting;
public class DefaultDataTracker implements DataTracker
{
diff --git a/harry-core/src/harry/runner/MutatingPartitionVisitor.java b/harry-core/src/harry/runner/MutatingPartitionVisitor.java
index 02aa6a1..4df793e 100644
--- a/harry-core/src/harry/runner/MutatingPartitionVisitor.java
+++ b/harry-core/src/harry/runner/MutatingPartitionVisitor.java
@@ -127,6 +127,7 @@ public class MutatingPartitionVisitor extends AbstractPartitionVisitor
if (sut.isShutdown())
throw new IllegalStateException("System under test is shut down");
+ // TODO: limit a number of retries
sut.executeAsync(statement.cql(), SystemUnderTest.ConsistencyLevel.QUORUM, statement.bindings())
.whenComplete((res, t) -> {
if (t != null)
diff --git a/harry-core/src/harry/runner/MutatingRowVisitor.java b/harry-core/src/harry/runner/MutatingRowVisitor.java
index b928df7..a14fc96 100644
--- a/harry-core/src/harry/runner/MutatingRowVisitor.java
+++ b/harry-core/src/harry/runner/MutatingRowVisitor.java
@@ -20,6 +20,7 @@ package harry.runner;
import harry.core.MetricReporter;
import harry.core.Run;
+import harry.core.VisibleForTesting;
import harry.ddl.SchemaSpec;
import harry.model.OpSelectors;
import harry.operations.CompiledStatement;
@@ -37,21 +38,35 @@ public class MutatingRowVisitor implements Operation
public MutatingRowVisitor(Run run)
{
- this.metricReporter = run.metricReporter;
- this.schema = run.schemaSpec;
- this.clock = run.clock;
- this.descriptorSelector = run.descriptorSelector;
- this.rangeSelector = run.rangeSelector;
+ this(run.schemaSpec,
+ run.clock,
+ run.descriptorSelector,
+ run.rangeSelector,
+ run.metricReporter);
}
- public CompiledStatement write(long lts, long pd, long cd, long opId)
+ @VisibleForTesting
+ public MutatingRowVisitor(SchemaSpec schema,
+ OpSelectors.MonotonicClock clock,
+ OpSelectors.DescriptorSelector descriptorSelector,
+ QueryGenerator rangeSelector,
+ MetricReporter metricReporter)
+ {
+ this.metricReporter = metricReporter;
+ this.schema = schema;
+ this.clock = clock;
+ this.descriptorSelector = descriptorSelector;
+ this.rangeSelector = rangeSelector;
+ }
+
+ public CompiledStatement insert(long lts, long pd, long cd, long opId)
{
metricReporter.insert();
long[] vds = descriptorSelector.vds(pd, cd, lts, opId, schema);
return WriteHelper.inflateInsert(schema, pd, cd, vds, null, clock.rts(lts));
}
- public CompiledStatement writeWithStatics(long lts, long pd, long cd, long opId)
+ public CompiledStatement insertWithStatics(long lts, long pd, long cd, long opId)
{
metricReporter.insert();
long[] vds = descriptorSelector.vds(pd, cd, lts, opId, schema);
@@ -59,6 +74,21 @@ public class MutatingRowVisitor implements Operation
return WriteHelper.inflateInsert(schema, pd, cd, vds, sds, clock.rts(lts));
}
+ public CompiledStatement update(long lts, long pd, long cd, long opId)
+ {
+ metricReporter.insert();
+ long[] vds = descriptorSelector.vds(pd, cd, lts, opId, schema);
+ return WriteHelper.inflateUpdate(schema, pd, cd, vds, null, clock.rts(lts));
+ }
+
+ public CompiledStatement updateWithStatics(long lts, long pd, long cd, long opId)
+ {
+ metricReporter.insert();
+ long[] vds = descriptorSelector.vds(pd, cd, lts, opId, schema);
+ long[] sds = descriptorSelector.sds(pd, cd, lts, opId, schema);
+ return WriteHelper.inflateUpdate(schema, pd, cd, vds, sds, clock.rts(lts));
+ }
+
public CompiledStatement deleteColumn(long lts, long pd, long cd, long opId)
{
metricReporter.columnDelete();
diff --git a/harry-core/src/harry/runner/Operation.java b/harry-core/src/harry/runner/Operation.java
index e56be4c..f44f3bd 100644
--- a/harry-core/src/harry/runner/Operation.java
+++ b/harry-core/src/harry/runner/Operation.java
@@ -35,12 +35,16 @@ public interface Operation
{
// TODO: switch to EnumMap
// TODO: pluggable capabilities; OperationKind can/should bear its own logic
- case WRITE:
- return write(lts, pd, cd, opId);
+ case INSERT:
+ return insert(lts, pd, cd, opId);
+ case UPDATE:
+ return update(lts, pd, cd, opId);
case DELETE_ROW:
return deleteRow(lts, pd, cd, opId);
- case WRITE_WITH_STATICS:
- return writeWithStatics(lts, pd, cd, opId);
+ case INSERT_WITH_STATICS:
+ return insertWithStatics(lts, pd, cd, opId);
+ case UPDATE_WITH_STATICS:
+ return updateWithStatics(lts, pd, cd, opId);
case DELETE_PARTITION:
return deletePartition(lts, pd, opId);
case DELETE_COLUMN:
@@ -56,7 +60,11 @@ public interface Operation
}
}
- CompiledStatement write(long lts, long pd, long cd, long opId);
+ CompiledStatement insert(long lts, long pd, long cd, long opId);
+ CompiledStatement update(long lts, long pd, long cd, long opId);
+
+ CompiledStatement insertWithStatics(long lts, long pd, long cd, long opId);
+ CompiledStatement updateWithStatics(long lts, long pd, long cd, long opId);
CompiledStatement deleteColumn(long lts, long pd, long cd, long opId);
@@ -66,8 +74,6 @@ public interface Operation
CompiledStatement deletePartition(long lts, long pd, long opId);
- CompiledStatement writeWithStatics(long lts, long pd, long cd, long opId);
-
CompiledStatement deleteRange(long lts, long pd, long opId);
CompiledStatement deleteSlice(long lts, long pd, long opId);
diff --git a/harry-core/src/harry/runner/QueryGenerator.java b/harry-core/src/harry/runner/QueryGenerator.java
index 65b21ac..829cf8b 100644
--- a/harry-core/src/harry/runner/QueryGenerator.java
+++ b/harry-core/src/harry/runner/QueryGenerator.java
@@ -317,7 +317,7 @@ public class QueryGenerator
// TODO: one of the ways to get rid of garbage here, and potentially even simplify the code is to
// simply return bounds here. After bounds are created, we slice them and generate query right
// from the bounds. In this case, we can even say that things like -inf/+inf are special values,
- // and use them as placeholdrs. Also, it'll be easier to manipulate relations.
+ // and use them as placeholders. Also, it'll be easier to manipulate relations.
return new Query.ClusteringRangeQuery(Query.QueryKind.CLUSTERING_RANGE,
pd,
stitchedMin,
diff --git a/harry-core/src/harry/util/BitSet.java b/harry-core/src/harry/util/BitSet.java
index 3c70052..4dc8823 100644
--- a/harry-core/src/harry/util/BitSet.java
+++ b/harry-core/src/harry/util/BitSet.java
@@ -178,7 +178,7 @@ public interface BitSet
public boolean isSet(int idx)
{
- assert idx < size();
+ assert idx < size() : String.format("Trying to query the bit (%s) outside the range of bitset (%s)", idx, size());
return BitSet.isSet(bits, idx);
}
diff --git a/harry-core/src/harry/util/TestRunner.java b/harry-core/src/harry/util/TestRunner.java
index 9abea62..4349fd2 100644
--- a/harry-core/src/harry/util/TestRunner.java
+++ b/harry-core/src/harry/util/TestRunner.java
@@ -44,19 +44,24 @@ public class TestRunner
public static <T1, T2> void test(Generator<T1> gen1,
Function<T1, Generator<T2>> gen2,
- Consumer<T2> validate)
+ ThrowingConsumer<T2> validate) throws Throwable
{
test(gen1,
(v1) -> test(gen2.apply(v1), validate));
}
public static <T1> void test(Generator<T1> gen1,
- Consumer<T1> validate)
+ ThrowingConsumer<T1> validate) throws Throwable
{
for (int i = 0; i < CYCLES; i++)
{
validate.accept(gen1.generate(rand));
}
}
+
+ public static interface ThrowingConsumer<T>
+ {
+ void accept(T t) throws Throwable;
+ }
}
diff --git a/harry-core/test/harry/model/OpSelectorsTest.java b/harry-core/test/harry/model/OpSelectorsTest.java
index b291cbc..266aeb9 100644
--- a/harry-core/test/harry/model/OpSelectorsTest.java
+++ b/harry-core/test/harry/model/OpSelectorsTest.java
@@ -25,7 +25,6 @@ import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
-import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
import java.util.function.BiConsumer;
@@ -39,7 +38,6 @@ import harry.core.Run;
import harry.ddl.ColumnSpec;
import harry.ddl.SchemaGenerators;
import harry.ddl.SchemaSpec;
-import harry.generators.RngUtils;
import harry.generators.Surjections;
import harry.generators.distribution.Distribution;
import harry.model.clock.OffsetClock;
@@ -186,10 +184,11 @@ public class OpSelectorsTest
OpSelectors.PdSelector pdSelector = new OpSelectors.DefaultPdSelector(rng, 10, 10);
OpSelectors.DescriptorSelector ckSelector = new OpSelectors.DefaultDescriptorSelector(rng,
new OpSelectors.ColumnSelectorBuilder().forAll(schema, Surjections.pick(BitSet.allUnset(0))).build(),
- OpSelectors.OperationSelector.weighted(Surjections.weights(10, 10, 80),
+ OpSelectors.OperationSelector.weighted(Surjections.weights(10, 10, 40, 40),
OpSelectors.OperationKind.DELETE_ROW,
OpSelectors.OperationKind.DELETE_COLUMN,
- OpSelectors.OperationKind.WRITE),
+ OpSelectors.OperationKind.INSERT,
+ OpSelectors.OperationKind.UPDATE),
new Distribution.ConstantDistribution(2),
new Distribution.ConstantDistribution(5),
10);
@@ -217,7 +216,13 @@ public class OpSelectorsTest
PartitionVisitor partitionVisitor = new MutatingPartitionVisitor(run,
(r) -> new Operation()
{
- public CompiledStatement write(long lts, long pd, long cd, long m)
+ public CompiledStatement insert(long lts, long pd, long cd, long m)
+ {
+ consumer.accept(pd, cd);
+ return compiledStatement;
+ }
+
+ public CompiledStatement update(long lts, long pd, long cd, long opId)
{
consumer.accept(pd, cd);
return compiledStatement;
@@ -247,7 +252,13 @@ public class OpSelectorsTest
return compiledStatement;
}
- public CompiledStatement writeWithStatics(long lts, long pd, long cd, long opId)
+ public CompiledStatement insertWithStatics(long lts, long pd, long cd, long opId)
+ {
+ consumer.accept(pd, cd);
+ return compiledStatement;
+ }
+
+ public CompiledStatement updateWithStatics(long lts, long pd, long cd, long opId)
{
consumer.accept(pd, cd);
return compiledStatement;
@@ -290,10 +301,11 @@ public class OpSelectorsTest
OpSelectors.DescriptorSelector ckSelector = new OpSelectors.HierarchicalDescriptorSelector(rng,
new int[] {10, 20},
OpSelectors.columnSelectorBuilder().forAll(schema, Surjections.pick(BitSet.allUnset(0))).build(),
- OpSelectors.OperationSelector.weighted(Surjections.weights(10, 10, 80),
+ OpSelectors.OperationSelector.weighted(Surjections.weights(10, 10, 40, 40),
OpSelectors.OperationKind.DELETE_ROW,
OpSelectors.OperationKind.DELETE_COLUMN,
- OpSelectors.OperationKind.WRITE),
+ OpSelectors.OperationKind.INSERT,
+ OpSelectors.OperationKind.UPDATE),
new Distribution.ConstantDistribution(2),
new Distribution.ConstantDistribution(5),
100);
@@ -323,8 +335,10 @@ public class OpSelectorsTest
config.put(OpSelectors.OperationKind.DELETE_COLUMN, 1);
config.put(OpSelectors.OperationKind.DELETE_PARTITION, 1);
config.put(OpSelectors.OperationKind.DELETE_COLUMN_WITH_STATICS, 1);
- config.put(OpSelectors.OperationKind.WRITE_WITH_STATICS, 1000);
- config.put(OpSelectors.OperationKind.WRITE, 1000);
+ config.put(OpSelectors.OperationKind.UPDATE, 500);
+ config.put(OpSelectors.OperationKind.INSERT, 500);
+ config.put(OpSelectors.OperationKind.UPDATE_WITH_STATICS, 500);
+ config.put(OpSelectors.OperationKind.INSERT_WITH_STATICS, 500);
int[] weights = new int[config.size()];
for (int i = 0; i < config.values().size(); i++)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra-harry] 03/05: Adjust config files
Posted by if...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-harry.git
commit 59604dbac36a9749c81c29e3739229c64237323c
Author: Alex Petrov <ol...@gmail.com>
AuthorDate: Tue Jul 13 10:41:18 2021 +0200
Adjust config files
---
README.md | 3 --
conf/{example.yaml => default.yaml} | 56 +++++++++++++++++++++++++++----------
conf/external.yaml | 56 +++++++++++++++++++++++++++----------
docker/Dockerfile.local | 2 +-
docker/run.sh | 2 +-
pom.xml | 24 ++++++----------
run-jvm.sh | 2 +-
7 files changed, 94 insertions(+), 51 deletions(-)
diff --git a/README.md b/README.md
index ec93b22..841e4ea 100644
--- a/README.md
+++ b/README.md
@@ -491,10 +491,8 @@ Harry is by no means feature-complete. Main things that are missing are:
* Some types (such as collections) are not deflatable
* Some types are implemented, but are not hooked up (`blob` and `text`) to DSL/generator
- * Partition deletions are not implemented
* 2i queries are not implemented
* Compact storage is not implemented
- * Static columns are not implemented
* Fault injection is not implemented
* Runner and scheduler are rather rudimentary and require significant rework and proper scheduling
* TTL is not supported
@@ -508,7 +506,6 @@ Some things, even though are implemented, can be improved or optimized:
* Inflated partition state and per-row operation log should be done in a compact
off-heap data structure
* Exhaustive checker can be significantly optimized
- * Harry shouldn't rely on java-driver for query generation
* Exhaustive checker should use more precise information from data tracker, not
just watermarks
* Decision-making about _when_ we visit partitions and/or rows should be improved
diff --git a/conf/example.yaml b/conf/default.yaml
similarity index 77%
rename from conf/example.yaml
rename to conf/default.yaml
index d28185d..cdbc34c 100644
--- a/conf/example.yaml
+++ b/conf/default.yaml
@@ -35,16 +35,6 @@ clock:
epoch_length: 1
epoch_time_unit: "SECONDS"
-# Runner is a is a component that schedules operations that change the cluster (system under test)
-# and model state.
-runner:
- concurrent:
- concurrency: 2
- partition_visitors:
- - mutating:
- row_visitor:
- mutating: {}
-
run_time: 2
run_time_unit: "HOURS"
@@ -73,17 +63,53 @@ clustering_descriptor_selector:
default:
modifications_per_lts:
type: "constant"
- constant: 10
+ constant: 4
rows_per_modification:
type: "constant"
- constant: 10
+ constant: 2
operation_kind_weights:
- WRITE: 97
DELETE_RANGE: 1
+ DELETE_SLICE: 1
DELETE_ROW: 1
DELETE_COLUMN: 1
+ DELETE_PARTITION: 1
+ DELETE_COLUMN_WITH_STATICS: 1
+ INSERT_WITH_STATICS: 50
+ INSERT: 50
+ UPDATE_WITH_STATICS: 50
+ UPDATE: 50
column_mask_bitsets: null
- max_partition_size: 100
+ max_partition_size: 1000
+
+# Runner is a is a component that schedules operations that change the cluster (system under test)
+# and model state.
+runner:
+ sequential:
+ partition_visitors:
+ - logging:
+ row_visitor:
+ mutating: {}
+ - sampler:
+ trigger_after: 1000
+ sample_partitions: 10
+ - parallel_validate_recent_partitions:
+ partition_count: 100
+ queries_per_partition: 2
+ concurrency: 20
+ trigger_after: 10000
+ model:
+ quiescent_checker: {}
+ - validate_all_partitions:
+ concurrency: 20
+ trigger_after: 100000
+ model:
+ quiescent_checker: {}
+
+# Model is responsible for tracking logical timestamps that
+data_tracker:
+ default:
+ max_seen_lts: -1
+ max_complete_lts: -1
metric_reporter:
- default: {}
\ No newline at end of file
+ no_op: {}
\ No newline at end of file
diff --git a/conf/external.yaml b/conf/external.yaml
index b4d8217..52943b8 100644
--- a/conf/external.yaml
+++ b/conf/external.yaml
@@ -35,16 +35,6 @@ clock:
epoch_length: 1
epoch_time_unit: "SECONDS"
-# Runner is a is a component that schedules operations that change the cluster (system under test)
-# and model state.
-runner:
- concurrent:
- concurrency: 2
- partition_visitors:
- - mutating:
- row_visitor:
- mutating: {}
-
run_time: 2
run_time_unit: "HOURS"
@@ -57,6 +47,12 @@ system_under_test:
username: null
password: null
+# Model is responsible for tracking logical timestamps that
+model:
+ exhaustive_checker:
+ max_seen_lts: 19
+ max_complete_lts: 16
+
# Partition descriptor selector controls how partitions is selected based on the current logical
# timestamp. Default implementation is a sliding window of partition descriptors that will visit
# one partition after the other in the window `slide_after_repeats` times. After that will
@@ -74,17 +70,47 @@ clustering_descriptor_selector:
default:
modifications_per_lts:
type: "constant"
- constant: 10
+ constant: 4
rows_per_modification:
type: "constant"
- constant: 10
+ constant: 2
operation_kind_weights:
- WRITE: 97
DELETE_RANGE: 1
+ DELETE_SLICE: 1
DELETE_ROW: 1
DELETE_COLUMN: 1
+ DELETE_PARTITION: 1
+ DELETE_COLUMN_WITH_STATICS: 1
+ INSERT_WITH_STATICS: 50
+ INSERT: 50
+ UPDATE_WITH_STATICS: 50
+ UPDATE: 50
column_mask_bitsets: null
- max_partition_size: 100
+ max_partition_size: 1000
+
+# Runner is a is a component that schedules operations that change the cluster (system under test)
+# and model state.
+runner:
+ sequential:
+ partition_visitors:
+ - logging:
+ row_visitor:
+ mutating: {}
+ - sampler:
+ trigger_after: 1000
+ sample_partitions: 10
+ - parallel_validate_recent_partitions:
+ partition_count: 100
+ queries_per_partition: 2
+ concurrency: 20
+ trigger_after: 10000
+ model:
+ quiescent_checker: {}
+ - validate_all_partitions:
+ concurrency: 20
+ trigger_after: 100000
+ model:
+ quiescent_checker: {}
metric_reporter:
- default: {}
+ no_op: {}
\ No newline at end of file
diff --git a/docker/Dockerfile.local b/docker/Dockerfile.local
index 21ce8fc..4b0032f 100644
--- a/docker/Dockerfile.local
+++ b/docker/Dockerfile.local
@@ -28,7 +28,7 @@ COPY ./harry-core/target/*.jar /opt/harry/lib/
COPY ./harry-integration/target/lib/ /opt/harry/lib/
COPY ./harry-integration/target/*.jar /opt/harry/
COPY ./test/conf/logback-dtest.xml /opt/harry/test/conf/logback-dtest.xml
-COPY ./conf/example.yaml /opt/harry/example.yaml
+COPY ./conf/default.yaml /opt/harry/default.yaml
COPY ./docker/run.sh /opt/harry/
WORKDIR /opt/harry
diff --git a/docker/run.sh b/docker/run.sh
index 490b21f..48afa0b 100755
--- a/docker/run.sh
+++ b/docker/run.sh
@@ -75,7 +75,7 @@ while true; do
-cp /opt/harry/lib/*:/opt/harry/harry-integration-0.0.1-SNAPSHOT.jar \
-Dharry.root=${HARRY_DIR} \
harry.runner.HarryRunnerJvm \
- /opt/harry/example.yaml
+ /opt/harry/default.yaml
if [ $? -ne 0 ]; then
if [ -e "failure.dump" ]; then
diff --git a/pom.xml b/pom.xml
index 7e23efd..276a670 100755
--- a/pom.xml
+++ b/pom.xml
@@ -52,7 +52,7 @@
<properties>
<javac.target>1.8</javac.target>
<harry.version>0.0.1-SNAPSHOT</harry.version>
- <cassandra.version>4.0.0-SNAPSHOT</cassandra.version>
+ <cassandra.version>4.1-58515c2de6</cassandra.version>
<jackson.version>2.11.3</jackson.version>
<dtest.version>0.0.7</dtest.version>
<jmh.version>1.11.3</jmh.version>
@@ -135,28 +135,22 @@
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>27.0-jre</version>
- </dependency>
-
- <dependency>
<groupId>org.apache.cassandra</groupId>
<artifactId>cassandra-dtest-shaded</artifactId>
<version>${cassandra.version}</version>
</dependency>
<dependency>
- <groupId>com.datastax.cassandra</groupId>
- <artifactId>cassandra-driver-core</artifactId>
- <version>3.6.0</version>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ <version>0.9.12</version>
</dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- <version>3.1</version>
- </dependency>
+ <dependency>
+ <groupId>com.datastax.cassandra</groupId>
+ <artifactId>cassandra-driver-core</artifactId>
+ <version>3.6.0</version>
+ </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
diff --git a/run-jvm.sh b/run-jvm.sh
index 6c330f8..e0999b5 100755
--- a/run-jvm.sh
+++ b/run-jvm.sh
@@ -46,4 +46,4 @@ java -ea \
-Dorg.apache.cassandra.test.logback.configurationFile=file://test/conf/logback-dtest.xml \
-cp harry-integration/target/harry-integration-0.0.1-SNAPSHOT.jar:$(find harry-integration/target/dependency/*.jar | tr -s '\n' ':'). \
harry.runner.HarryRunnerJvm \
- conf/example.yaml
\ No newline at end of file
+ conf/default.yaml
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra-harry] 04/05: Move classes to appropriate packages
Posted by if...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-harry.git
commit 83dd5a64edf9f62f5b3c5c468c6435f49b580cf3
Author: Alex Petrov <ol...@gmail.com>
AuthorDate: Thu Jul 15 08:29:57 2021 +0200
Move classes to appropriate packages
---
harry-core/src/harry/core/Configuration.java | 20 ++++----
harry-core/src/harry/core/Run.java | 2 +-
.../src/harry/corruptor/AddExtraRowCorruptor.java | 2 +-
.../harry/corruptor/QueryResponseCorruptor.java | 3 +-
harry-core/src/harry/model/Model.java | 3 +-
harry-core/src/harry/model/NoOpChecker.java | 2 +-
harry-core/src/harry/model/QuiescentChecker.java | 4 +-
harry-core/src/harry/model/SelectHelper.java | 2 +-
.../src/harry/{runner => operations}/Query.java | 5 +-
.../{runner => operations}/QueryGenerator.java | 4 +-
harry-core/src/harry/reconciler/Reconciler.java | 8 ++--
harry-core/src/harry/runner/Runner.java | 2 +-
.../AbstractPartitionVisitor.java | 7 ++-
.../AllPartitionsValidator.java | 3 +-
.../CorruptingPartitionVisitor.java | 4 +-
.../visitors}/FaultInjectingPartitionVisitor.java | 4 +-
.../LoggingPartitionVisitor.java | 2 +-
.../MutatingPartitionVisitor.java | 4 +-
.../{runner => visitors}/MutatingRowVisitor.java | 4 +-
.../src/harry/{runner => visitors}/Operation.java | 2 +-
.../ParallelRecentPartitionValidator.java | 4 +-
.../{runner => visitors}/ParallelValidator.java | 2 +-
.../{runner => visitors}/PartitionVisitor.java | 2 +-
.../RecentPartitionValidator.java | 6 +--
.../src/harry/{runner => visitors}/Sampler.java | 2 +-
.../SinglePartitionValidator.java | 5 +-
harry-core/test/harry/model/OpSelectorsTest.java | 6 +--
harry-core/test/harry/operations/RelationTest.java | 2 -
.../dependency-reduced-pom.xml | 54 ++++++++++++++++++++++
.../src/harry/runner/QueryingNoOpChecker.java | 1 +
.../harry/runner/RepairingLocalStateValidator.java | 3 ++
harry-integration/src/harry/runner/Reproduce.java | 1 +
.../src/harry/runner/TrivialShrinker.java | 53 ++-------------------
.../harry/visitors/SkippingPartitionVisitor.java | 53 +++++++++++++++++++++
.../test/harry/ddl/SchemaGenTest.java | 2 +-
.../generators/DataGeneratorsIntegrationTest.java | 8 ++--
.../test/harry/model/ModelTestBase.java | 8 ++--
.../harry/model/QuerySelectorNegativeTest.java | 10 ++--
.../test/harry/model/QuerySelectorTest.java | 10 ++--
.../model/QuiescentCheckerIntegrationTest.java | 6 +--
.../test/harry/model/TestEveryClustering.java | 24 ++++------
.../test/harry/op/RowVisitorTest.java | 2 +-
42 files changed, 206 insertions(+), 145 deletions(-)
diff --git a/harry-core/src/harry/core/Configuration.java b/harry-core/src/harry/core/Configuration.java
index 4a2a1e6..0e065d5 100644
--- a/harry-core/src/harry/core/Configuration.java
+++ b/harry-core/src/harry/core/Configuration.java
@@ -46,19 +46,19 @@ import harry.model.clock.ApproximateMonotonicClock;
import harry.model.clock.OffsetClock;
import harry.model.sut.PrintlnSut;
import harry.model.sut.SystemUnderTest;
-import harry.runner.AllPartitionsValidator;
-import harry.runner.CorruptingPartitionVisitor;
+import harry.visitors.AllPartitionsValidator;
+import harry.visitors.CorruptingPartitionVisitor;
import harry.runner.DataTracker;
import harry.runner.DefaultDataTracker;
-import harry.runner.LoggingPartitionVisitor;
-import harry.runner.MutatingPartitionVisitor;
-import harry.runner.MutatingRowVisitor;
-import harry.runner.Operation;
-import harry.runner.ParallelRecentPartitionValidator;
-import harry.runner.PartitionVisitor;
-import harry.runner.RecentPartitionValidator;
+import harry.visitors.LoggingPartitionVisitor;
+import harry.visitors.MutatingPartitionVisitor;
+import harry.visitors.MutatingRowVisitor;
+import harry.visitors.Operation;
+import harry.visitors.ParallelRecentPartitionValidator;
+import harry.visitors.PartitionVisitor;
+import harry.visitors.RecentPartitionValidator;
import harry.runner.Runner;
-import harry.runner.Sampler;
+import harry.visitors.Sampler;
import harry.util.BitSet;
public class Configuration
diff --git a/harry-core/src/harry/core/Run.java b/harry-core/src/harry/core/Run.java
index d31e7ad..b0c8afc 100644
--- a/harry-core/src/harry/core/Run.java
+++ b/harry-core/src/harry/core/Run.java
@@ -22,7 +22,7 @@ import harry.ddl.SchemaSpec;
import harry.model.OpSelectors;
import harry.model.sut.SystemUnderTest;
import harry.runner.DataTracker;
-import harry.runner.QueryGenerator;
+import harry.operations.QueryGenerator;
public class Run
{
diff --git a/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java b/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java
index 65f4a0d..e05dffd 100644
--- a/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java
+++ b/harry-core/src/harry/corruptor/AddExtraRowCorruptor.java
@@ -30,7 +30,7 @@ import harry.model.OpSelectors;
import harry.model.SelectHelper;
import harry.model.sut.SystemUnderTest;
import harry.operations.WriteHelper;
-import harry.runner.Query;
+import harry.operations.Query;
public class AddExtraRowCorruptor implements QueryResponseCorruptor
{
diff --git a/harry-core/src/harry/corruptor/QueryResponseCorruptor.java b/harry-core/src/harry/corruptor/QueryResponseCorruptor.java
index f4a3778..62bf589 100644
--- a/harry-core/src/harry/corruptor/QueryResponseCorruptor.java
+++ b/harry-core/src/harry/corruptor/QueryResponseCorruptor.java
@@ -31,8 +31,7 @@ import harry.model.OpSelectors;
import harry.model.SelectHelper;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
-import harry.runner.HarryRunner;
-import harry.runner.Query;
+import harry.operations.Query;
public interface QueryResponseCorruptor
{
diff --git a/harry-core/src/harry/model/Model.java b/harry-core/src/harry/model/Model.java
index d6ab865..1521d0f 100644
--- a/harry-core/src/harry/model/Model.java
+++ b/harry-core/src/harry/model/Model.java
@@ -19,8 +19,7 @@
package harry.model;
import harry.core.Run;
-import harry.reconciler.Reconciler;
-import harry.runner.Query;
+import harry.operations.Query;
public interface Model
{
diff --git a/harry-core/src/harry/model/NoOpChecker.java b/harry-core/src/harry/model/NoOpChecker.java
index a13b6ec..10f0a4a 100644
--- a/harry-core/src/harry/model/NoOpChecker.java
+++ b/harry-core/src/harry/model/NoOpChecker.java
@@ -20,7 +20,7 @@ package harry.model;
import harry.core.Run;
import harry.model.sut.SystemUnderTest;
-import harry.runner.Query;
+import harry.operations.Query;
public class NoOpChecker implements Model
{
diff --git a/harry-core/src/harry/model/QuiescentChecker.java b/harry-core/src/harry/model/QuiescentChecker.java
index a48ccce..d17d6f4 100644
--- a/harry-core/src/harry/model/QuiescentChecker.java
+++ b/harry-core/src/harry/model/QuiescentChecker.java
@@ -24,15 +24,13 @@ import java.util.Iterator;
import java.util.List;
import java.util.function.Supplier;
-import harry.core.Configuration;
import harry.core.Run;
import harry.data.ResultSetRow;
import harry.ddl.SchemaSpec;
import harry.model.sut.SystemUnderTest;
import harry.reconciler.Reconciler;
import harry.runner.DataTracker;
-import harry.runner.Query;
-import harry.runner.QueryGenerator;
+import harry.operations.Query;
import static harry.generators.DataGenerators.NIL_DESCR;
diff --git a/harry-core/src/harry/model/SelectHelper.java b/harry-core/src/harry/model/SelectHelper.java
index fc8f6f7..c0813ba 100644
--- a/harry-core/src/harry/model/SelectHelper.java
+++ b/harry-core/src/harry/model/SelectHelper.java
@@ -28,7 +28,7 @@ import harry.ddl.SchemaSpec;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
import harry.operations.Relation;
-import harry.runner.Query;
+import harry.operations.Query;
public class SelectHelper
{
diff --git a/harry-core/src/harry/runner/Query.java b/harry-core/src/harry/operations/Query.java
similarity index 98%
rename from harry-core/src/harry/runner/Query.java
rename to harry-core/src/harry/operations/Query.java
index 1eef505..d7e75c3 100644
--- a/harry-core/src/harry/runner/Query.java
+++ b/harry-core/src/harry/operations/Query.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.operations;
import java.util.ArrayList;
import java.util.Arrays;
@@ -30,9 +30,6 @@ import org.slf4j.LoggerFactory;
import harry.ddl.SchemaSpec;
import harry.model.SelectHelper;
-import harry.operations.CompiledStatement;
-import harry.operations.DeleteHelper;
-import harry.operations.Relation;
import harry.util.Ranges;
import static harry.operations.Relation.FORWARD_COMPARATOR;
diff --git a/harry-core/src/harry/runner/QueryGenerator.java b/harry-core/src/harry/operations/QueryGenerator.java
similarity index 99%
rename from harry-core/src/harry/runner/QueryGenerator.java
rename to harry-core/src/harry/operations/QueryGenerator.java
index 829cf8b..6c815b5 100644
--- a/harry-core/src/harry/runner/QueryGenerator.java
+++ b/harry-core/src/harry/operations/QueryGenerator.java
@@ -16,10 +16,9 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.operations;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.function.LongSupplier;
@@ -34,7 +33,6 @@ import harry.generators.DataGenerators;
import harry.generators.RngUtils;
import harry.generators.Surjections;
import harry.model.OpSelectors;
-import harry.operations.Relation;
// TODO: there's a lot of potential to reduce an amount of garbage here.
// TODO: refactor. Currently, this class is a base for both SELECT and DELETE statements. In retrospect,
diff --git a/harry-core/src/harry/reconciler/Reconciler.java b/harry-core/src/harry/reconciler/Reconciler.java
index a9709b3..da2daa7 100644
--- a/harry-core/src/harry/reconciler/Reconciler.java
+++ b/harry-core/src/harry/reconciler/Reconciler.java
@@ -34,10 +34,10 @@ import harry.core.Run;
import harry.ddl.ColumnSpec;
import harry.ddl.SchemaSpec;
import harry.model.OpSelectors;
-import harry.runner.AbstractPartitionVisitor;
-import harry.runner.PartitionVisitor;
-import harry.runner.Query;
-import harry.runner.QueryGenerator;
+import harry.visitors.AbstractPartitionVisitor;
+import harry.visitors.PartitionVisitor;
+import harry.operations.Query;
+import harry.operations.QueryGenerator;
import harry.util.BitSet;
import harry.util.Ranges;
diff --git a/harry-core/src/harry/runner/Runner.java b/harry-core/src/harry/runner/Runner.java
index 06bc481..b172723 100644
--- a/harry-core/src/harry/runner/Runner.java
+++ b/harry-core/src/harry/runner/Runner.java
@@ -28,7 +28,6 @@ import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -40,6 +39,7 @@ import org.slf4j.LoggerFactory;
import harry.core.Configuration;
import harry.core.Run;
import harry.model.OpSelectors;
+import harry.visitors.PartitionVisitor;
public abstract class Runner
diff --git a/harry-core/src/harry/runner/AbstractPartitionVisitor.java b/harry-core/src/harry/visitors/AbstractPartitionVisitor.java
similarity index 93%
rename from harry-core/src/harry/runner/AbstractPartitionVisitor.java
rename to harry-core/src/harry/visitors/AbstractPartitionVisitor.java
index 0455cbe..3f58e3a 100644
--- a/harry-core/src/harry/runner/AbstractPartitionVisitor.java
+++ b/harry-core/src/harry/visitors/AbstractPartitionVisitor.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import harry.ddl.SchemaSpec;
import harry.model.OpSelectors;
@@ -27,6 +27,11 @@ public abstract class AbstractPartitionVisitor implements PartitionVisitor
protected final OpSelectors.DescriptorSelector descriptorSelector;
protected final SchemaSpec schema;
+ public AbstractPartitionVisitor(AbstractPartitionVisitor visitor)
+ {
+ this(visitor.pdSelector, visitor.descriptorSelector, visitor.schema);
+ }
+
public AbstractPartitionVisitor(OpSelectors.PdSelector pdSelector,
OpSelectors.DescriptorSelector descriptorSelector,
SchemaSpec schema)
diff --git a/harry-core/src/harry/runner/AllPartitionsValidator.java b/harry-core/src/harry/visitors/AllPartitionsValidator.java
similarity index 98%
rename from harry-core/src/harry/runner/AllPartitionsValidator.java
rename to harry-core/src/harry/visitors/AllPartitionsValidator.java
index f42ab65..5a5a1c7 100644
--- a/harry-core/src/harry/runner/AllPartitionsValidator.java
+++ b/harry-core/src/harry/visitors/AllPartitionsValidator.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
@@ -34,6 +34,7 @@ import harry.ddl.SchemaSpec;
import harry.model.Model;
import harry.model.OpSelectors;
import harry.model.sut.SystemUnderTest;
+import harry.operations.Query;
// This might be something that potentially grows into the validator described in the design doc;
// right now it's just a helper/container class
diff --git a/harry-core/src/harry/runner/CorruptingPartitionVisitor.java b/harry-core/src/harry/visitors/CorruptingPartitionVisitor.java
similarity index 97%
rename from harry-core/src/harry/runner/CorruptingPartitionVisitor.java
rename to harry-core/src/harry/visitors/CorruptingPartitionVisitor.java
index d079cf3..71c425b 100644
--- a/harry-core/src/harry/runner/CorruptingPartitionVisitor.java
+++ b/harry-core/src/harry/visitors/CorruptingPartitionVisitor.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import java.util.Random;
import java.util.concurrent.atomic.AtomicLong;
@@ -29,6 +29,8 @@ import harry.corruptor.ChangeValueCorruptor;
import harry.corruptor.HideRowCorruptor;
import harry.corruptor.HideValueCorruptor;
import harry.corruptor.QueryResponseCorruptor;
+import harry.runner.HarryRunner;
+import harry.operations.Query;
public class CorruptingPartitionVisitor implements PartitionVisitor
{
diff --git a/harry-integration/src/harry/runner/FaultInjectingPartitionVisitor.java b/harry-core/src/harry/visitors/FaultInjectingPartitionVisitor.java
similarity index 97%
rename from harry-integration/src/harry/runner/FaultInjectingPartitionVisitor.java
rename to harry-core/src/harry/visitors/FaultInjectingPartitionVisitor.java
index a6483bc..6a03d35 100644
--- a/harry-integration/src/harry/runner/FaultInjectingPartitionVisitor.java
+++ b/harry-core/src/harry/visitors/FaultInjectingPartitionVisitor.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
@@ -27,8 +27,6 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import harry.core.Configuration;
import harry.core.Run;
-import harry.model.sut.InJvmSut;
-import harry.model.sut.MixedVersionInJvmSut;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
diff --git a/harry-core/src/harry/runner/LoggingPartitionVisitor.java b/harry-core/src/harry/visitors/LoggingPartitionVisitor.java
similarity index 99%
rename from harry-core/src/harry/runner/LoggingPartitionVisitor.java
rename to harry-core/src/harry/visitors/LoggingPartitionVisitor.java
index 9cb827d..3e97ba3 100644
--- a/harry-core/src/harry/runner/LoggingPartitionVisitor.java
+++ b/harry-core/src/harry/visitors/LoggingPartitionVisitor.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import java.io.BufferedWriter;
import java.io.File;
diff --git a/harry-core/src/harry/runner/MutatingPartitionVisitor.java b/harry-core/src/harry/visitors/MutatingPartitionVisitor.java
similarity index 98%
rename from harry-core/src/harry/runner/MutatingPartitionVisitor.java
rename to harry-core/src/harry/visitors/MutatingPartitionVisitor.java
index 4df793e..88456fe 100644
--- a/harry-core/src/harry/runner/MutatingPartitionVisitor.java
+++ b/harry-core/src/harry/visitors/MutatingPartitionVisitor.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import java.util.ArrayList;
import java.util.List;
@@ -29,10 +29,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import harry.core.Run;
-import harry.model.Model;
import harry.model.OpSelectors;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
+import harry.runner.DataTracker;
public class MutatingPartitionVisitor extends AbstractPartitionVisitor
{
diff --git a/harry-core/src/harry/runner/MutatingRowVisitor.java b/harry-core/src/harry/visitors/MutatingRowVisitor.java
similarity index 98%
rename from harry-core/src/harry/runner/MutatingRowVisitor.java
rename to harry-core/src/harry/visitors/MutatingRowVisitor.java
index a14fc96..0c5db9a 100644
--- a/harry-core/src/harry/runner/MutatingRowVisitor.java
+++ b/harry-core/src/harry/visitors/MutatingRowVisitor.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import harry.core.MetricReporter;
import harry.core.Run;
@@ -26,6 +26,8 @@ import harry.model.OpSelectors;
import harry.operations.CompiledStatement;
import harry.operations.DeleteHelper;
import harry.operations.WriteHelper;
+import harry.operations.Query;
+import harry.operations.QueryGenerator;
import harry.util.BitSet;
public class MutatingRowVisitor implements Operation
diff --git a/harry-core/src/harry/runner/Operation.java b/harry-core/src/harry/visitors/Operation.java
similarity index 99%
rename from harry-core/src/harry/runner/Operation.java
rename to harry-core/src/harry/visitors/Operation.java
index f44f3bd..1af21fc 100644
--- a/harry-core/src/harry/runner/Operation.java
+++ b/harry-core/src/harry/visitors/Operation.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import harry.core.Run;
import harry.model.OpSelectors;
diff --git a/harry-core/src/harry/runner/ParallelRecentPartitionValidator.java b/harry-core/src/harry/visitors/ParallelRecentPartitionValidator.java
similarity index 98%
rename from harry-core/src/harry/runner/ParallelRecentPartitionValidator.java
rename to harry-core/src/harry/visitors/ParallelRecentPartitionValidator.java
index b363baf..a1688cd 100644
--- a/harry-core/src/harry/runner/ParallelRecentPartitionValidator.java
+++ b/harry-core/src/harry/visitors/ParallelRecentPartitionValidator.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import java.io.BufferedWriter;
import java.io.File;
@@ -38,6 +38,8 @@ import harry.core.Configuration;
import harry.core.Run;
import harry.generators.Surjections;
import harry.model.Model;
+import harry.operations.Query;
+import harry.operations.QueryGenerator;
public class ParallelRecentPartitionValidator extends ParallelValidator<ParallelRecentPartitionValidator.State>
{
diff --git a/harry-core/src/harry/runner/ParallelValidator.java b/harry-core/src/harry/visitors/ParallelValidator.java
similarity index 99%
rename from harry-core/src/harry/runner/ParallelValidator.java
rename to harry-core/src/harry/visitors/ParallelValidator.java
index 742a7cc..2964eb7 100644
--- a/harry-core/src/harry/runner/ParallelValidator.java
+++ b/harry-core/src/harry/visitors/ParallelValidator.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
diff --git a/harry-core/src/harry/runner/PartitionVisitor.java b/harry-core/src/harry/visitors/PartitionVisitor.java
similarity index 97%
rename from harry-core/src/harry/runner/PartitionVisitor.java
rename to harry-core/src/harry/visitors/PartitionVisitor.java
index 85e3b26..77de711 100644
--- a/harry-core/src/harry/runner/PartitionVisitor.java
+++ b/harry-core/src/harry/visitors/PartitionVisitor.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import harry.core.Run;
diff --git a/harry-core/src/harry/runner/RecentPartitionValidator.java b/harry-core/src/harry/visitors/RecentPartitionValidator.java
similarity index 97%
rename from harry-core/src/harry/runner/RecentPartitionValidator.java
rename to harry-core/src/harry/visitors/RecentPartitionValidator.java
index 69388dd..4bb1e61 100644
--- a/harry-core/src/harry/runner/RecentPartitionValidator.java
+++ b/harry-core/src/harry/visitors/RecentPartitionValidator.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import java.io.BufferedWriter;
import java.io.File;
@@ -24,7 +24,6 @@ import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
-import java.util.Arrays;
import java.util.concurrent.atomic.AtomicLong;
import org.slf4j.Logger;
@@ -35,7 +34,8 @@ import harry.core.Run;
import harry.generators.Surjections;
import harry.model.Model;
import harry.model.OpSelectors;
-import harry.operations.CompiledStatement;
+import harry.operations.Query;
+import harry.operations.QueryGenerator;
public class RecentPartitionValidator implements PartitionVisitor
{
diff --git a/harry-core/src/harry/runner/Sampler.java b/harry-core/src/harry/visitors/Sampler.java
similarity index 99%
rename from harry-core/src/harry/runner/Sampler.java
rename to harry-core/src/harry/visitors/Sampler.java
index c362bd9..e4088d9 100644
--- a/harry-core/src/harry/runner/Sampler.java
+++ b/harry-core/src/harry/visitors/Sampler.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import java.util.concurrent.atomic.AtomicLong;
diff --git a/harry-core/src/harry/runner/SinglePartitionValidator.java b/harry-core/src/harry/visitors/SinglePartitionValidator.java
similarity index 94%
rename from harry-core/src/harry/runner/SinglePartitionValidator.java
rename to harry-core/src/harry/visitors/SinglePartitionValidator.java
index febfc6a..313cff9 100644
--- a/harry-core/src/harry/runner/SinglePartitionValidator.java
+++ b/harry-core/src/harry/visitors/SinglePartitionValidator.java
@@ -16,10 +16,13 @@
* limitations under the License.
*/
-package harry.runner;
+package harry.visitors;
import harry.core.Run;
import harry.model.Model;
+import harry.operations.Query;
+import harry.operations.QueryGenerator;
+import harry.visitors.PartitionVisitor;
public class SinglePartitionValidator implements PartitionVisitor
{
diff --git a/harry-core/test/harry/model/OpSelectorsTest.java b/harry-core/test/harry/model/OpSelectorsTest.java
index 266aeb9..709ba81 100644
--- a/harry-core/test/harry/model/OpSelectorsTest.java
+++ b/harry-core/test/harry/model/OpSelectorsTest.java
@@ -44,9 +44,9 @@ import harry.model.clock.OffsetClock;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
import harry.runner.DataTracker;
-import harry.runner.MutatingPartitionVisitor;
-import harry.runner.PartitionVisitor;
-import harry.runner.Operation;
+import harry.visitors.MutatingPartitionVisitor;
+import harry.visitors.PartitionVisitor;
+import harry.visitors.Operation;
import harry.util.BitSet;
public class OpSelectorsTest
diff --git a/harry-core/test/harry/operations/RelationTest.java b/harry-core/test/harry/operations/RelationTest.java
index 122b224..e0c6e73 100644
--- a/harry-core/test/harry/operations/RelationTest.java
+++ b/harry-core/test/harry/operations/RelationTest.java
@@ -31,8 +31,6 @@ import harry.ddl.ColumnSpec;
import harry.ddl.SchemaSpec;
import harry.generators.DataGeneratorsTest;
import harry.model.OpSelectors;
-import harry.runner.Query;
-import harry.runner.QueryGenerator;
import harry.util.BitSet;
public class RelationTest
diff --git a/harry-integration-external/dependency-reduced-pom.xml b/harry-integration-external/dependency-reduced-pom.xml
new file mode 100644
index 0000000..e2e00a5
--- /dev/null
+++ b/harry-integration-external/dependency-reduced-pom.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <parent>
+ <artifactId>harry-parent</artifactId>
+ <groupId>org.apache.cassandra</groupId>
+ <version>0.0.1-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>harry-integration-external</artifactId>
+ <name>Harry Integration - External</name>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.2.4</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <transformers>
+ <transformer>
+ <mainClass>harry.runner.external.HarryRunnerExternal</mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>hamcrest-core</artifactId>
+ <groupId>org.hamcrest</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.quicktheories</groupId>
+ <artifactId>quicktheories</artifactId>
+ <version>0.25</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/harry-integration/src/harry/runner/QueryingNoOpChecker.java b/harry-integration/src/harry/runner/QueryingNoOpChecker.java
index 08b8e6b..0e7d4fb 100644
--- a/harry-integration/src/harry/runner/QueryingNoOpChecker.java
+++ b/harry-integration/src/harry/runner/QueryingNoOpChecker.java
@@ -25,6 +25,7 @@ import harry.core.Run;
import harry.model.Model;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
+import harry.operations.Query;
public class QueryingNoOpChecker implements Model
{
diff --git a/harry-integration/src/harry/runner/RepairingLocalStateValidator.java b/harry-integration/src/harry/runner/RepairingLocalStateValidator.java
index 98beb93..41926f0 100644
--- a/harry-integration/src/harry/runner/RepairingLocalStateValidator.java
+++ b/harry-integration/src/harry/runner/RepairingLocalStateValidator.java
@@ -32,6 +32,9 @@ import harry.model.QuiescentChecker;
import harry.model.sut.InJvmSut;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
+import harry.operations.Query;
+import harry.visitors.AllPartitionsValidator;
+import harry.visitors.PartitionVisitor;
import static harry.model.SelectHelper.resultSetToRow;
diff --git a/harry-integration/src/harry/runner/Reproduce.java b/harry-integration/src/harry/runner/Reproduce.java
index f3ae9cb..5d34ecd 100644
--- a/harry-integration/src/harry/runner/Reproduce.java
+++ b/harry-integration/src/harry/runner/Reproduce.java
@@ -23,6 +23,7 @@ import java.io.File;
import harry.core.Configuration;
import harry.core.Run;
import harry.model.sut.PrintlnSut;
+import harry.operations.Query;
import harry.reconciler.Reconciler;
public class Reproduce
diff --git a/harry-integration/src/harry/runner/TrivialShrinker.java b/harry-integration/src/harry/runner/TrivialShrinker.java
index 82e0572..23bdbcc 100644
--- a/harry-integration/src/harry/runner/TrivialShrinker.java
+++ b/harry-integration/src/harry/runner/TrivialShrinker.java
@@ -28,6 +28,9 @@ import java.util.function.Predicate;
import harry.core.Configuration;
import harry.core.Run;
+import harry.visitors.AbstractPartitionVisitor;
+import harry.visitors.PartitionVisitor;
+import harry.visitors.SkippingPartitionVisitor;
/**
* A most trivial imaginable shrinker: attempts to skip partitions and/or logical timestamps to see if the
@@ -167,56 +170,6 @@ public class TrivialShrinker
}
}
- public static class SkippingPartitionVisitor extends AbstractPartitionVisitor
- {
- private final AbstractPartitionVisitor delegate;
- private final Set<Long> ltsToSkip;
- private final Set<Long> pdsToSkip;
-
- public SkippingPartitionVisitor(AbstractPartitionVisitor delegate,
- Set<Long> ltsToSkip,
- Set<Long> pdsToSkip)
- {
- super(delegate.pdSelector, delegate.descriptorSelector, delegate.schema);
- this.delegate = delegate;
- this.ltsToSkip = ltsToSkip;
- this.pdsToSkip = pdsToSkip;
- }
-
- protected void beforeLts(long lts, long pd)
- {
- delegate.beforeLts(lts, pd);
- }
-
- protected void afterLts(long lts, long pd)
- {
- delegate.afterLts(lts, pd);
- }
-
- protected void beforeBatch(long lts, long pd, long m)
- {
- delegate.beforeBatch(lts, pd, m);
- }
-
- protected void operation(long lts, long pd, long cd, long m, long opId)
- {
- if (pdsToSkip.contains(pd) || ltsToSkip.contains(lts))
- return;
-
- delegate.operation(lts, pd, cd, m, opId);
- }
-
- protected void afterBatch(long lts, long pd, long m)
- {
- delegate.afterBatch(lts, pd, m);
- }
-
- public void shutdown() throws InterruptedException
- {
- delegate.shutdown();
- }
- }
-
public static String toString(Set<Long> longs)
{
if (longs.isEmpty())
diff --git a/harry-integration/src/harry/visitors/SkippingPartitionVisitor.java b/harry-integration/src/harry/visitors/SkippingPartitionVisitor.java
new file mode 100644
index 0000000..fa910cb
--- /dev/null
+++ b/harry-integration/src/harry/visitors/SkippingPartitionVisitor.java
@@ -0,0 +1,53 @@
+package harry.visitors;
+
+import java.util.Set;
+
+public class SkippingPartitionVisitor extends AbstractPartitionVisitor
+{
+ private final AbstractPartitionVisitor delegate;
+ private final Set<Long> ltsToSkip;
+ private final Set<Long> pdsToSkip;
+
+ public SkippingPartitionVisitor(AbstractPartitionVisitor delegate,
+ Set<Long> ltsToSkip,
+ Set<Long> pdsToSkip)
+ {
+ super(delegate);
+ this.delegate = delegate;
+ this.ltsToSkip = ltsToSkip;
+ this.pdsToSkip = pdsToSkip;
+ }
+
+ protected void beforeLts(long lts, long pd)
+ {
+ delegate.beforeLts(lts, pd);
+ }
+
+ protected void afterLts(long lts, long pd)
+ {
+ delegate.afterLts(lts, pd);
+ }
+
+ protected void beforeBatch(long lts, long pd, long m)
+ {
+ delegate.beforeBatch(lts, pd, m);
+ }
+
+ protected void operation(long lts, long pd, long cd, long m, long opId)
+ {
+ if (pdsToSkip.contains(pd) || ltsToSkip.contains(lts))
+ return;
+
+ delegate.operation(lts, pd, cd, m, opId);
+ }
+
+ protected void afterBatch(long lts, long pd, long m)
+ {
+ delegate.afterBatch(lts, pd, m);
+ }
+
+ public void shutdown() throws InterruptedException
+ {
+ delegate.shutdown();
+ }
+}
diff --git a/harry-integration/test/harry/ddl/SchemaGenTest.java b/harry-integration/test/harry/ddl/SchemaGenTest.java
index 2f7e898..c6204cd 100644
--- a/harry-integration/test/harry/ddl/SchemaGenTest.java
+++ b/harry-integration/test/harry/ddl/SchemaGenTest.java
@@ -32,7 +32,7 @@ import harry.QuickTheoriesAdapter;
import harry.generators.Generator;
import harry.operations.CompiledStatement;
-import harry.runner.Query;
+import harry.operations.Query;
import harry.util.TestRunner;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.Keyspace;
diff --git a/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java b/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
index cadee0c..b819eb5 100644
--- a/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
+++ b/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
@@ -32,10 +32,10 @@ import harry.generators.distribution.Distribution;
import harry.model.NoOpChecker;
import harry.model.OpSelectors;
import harry.model.sut.SystemUnderTest;
-import harry.runner.MutatingPartitionVisitor;
-import harry.runner.MutatingRowVisitor;
-import harry.runner.PartitionVisitor;
-import harry.runner.SinglePartitionValidator;
+import harry.visitors.MutatingPartitionVisitor;
+import harry.visitors.MutatingRowVisitor;
+import harry.visitors.PartitionVisitor;
+import harry.visitors.SinglePartitionValidator;
import harry.util.TestRunner;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.UntypedResultSet;
diff --git a/harry-integration/test/harry/model/ModelTestBase.java b/harry-integration/test/harry/model/ModelTestBase.java
index 5d1659d..70b9fb3 100644
--- a/harry-integration/test/harry/model/ModelTestBase.java
+++ b/harry-integration/test/harry/model/ModelTestBase.java
@@ -28,11 +28,11 @@ import harry.core.Configuration;
import harry.core.Run;
import harry.ddl.SchemaGenerators;
import harry.ddl.SchemaSpec;
-import harry.runner.LoggingPartitionVisitor;
-import harry.runner.MutatingRowVisitor;
-import harry.runner.PartitionVisitor;
+import harry.visitors.LoggingPartitionVisitor;
+import harry.visitors.MutatingRowVisitor;
+import harry.visitors.PartitionVisitor;
import harry.runner.Runner;
-import harry.runner.SinglePartitionValidator;
+import harry.visitors.SinglePartitionValidator;
public abstract class ModelTestBase extends IntegrationTestBase
{
diff --git a/harry-integration/test/harry/model/QuerySelectorNegativeTest.java b/harry-integration/test/harry/model/QuerySelectorNegativeTest.java
index 8c635c8..928e4e9 100644
--- a/harry-integration/test/harry/model/QuerySelectorNegativeTest.java
+++ b/harry-integration/test/harry/model/QuerySelectorNegativeTest.java
@@ -25,6 +25,7 @@ import java.util.Map;
import java.util.Random;
import java.util.function.Supplier;
+import harry.operations.Query;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -39,11 +40,10 @@ import harry.corruptor.HideValueCorruptor;
import harry.corruptor.QueryResponseCorruptor;
import harry.corruptor.ShowValueCorruptor;
import harry.ddl.SchemaGenerators;
-import harry.runner.MutatingPartitionVisitor;
-import harry.runner.MutatingRowVisitor;
-import harry.runner.PartitionVisitor;
-import harry.runner.Query;
-import harry.runner.QueryGenerator;
+import harry.visitors.MutatingPartitionVisitor;
+import harry.visitors.MutatingRowVisitor;
+import harry.visitors.PartitionVisitor;
+import harry.operations.QueryGenerator;
import static harry.corruptor.QueryResponseCorruptor.SimpleQueryResponseCorruptor;
diff --git a/harry-integration/test/harry/model/QuerySelectorTest.java b/harry-integration/test/harry/model/QuerySelectorTest.java
index 484d14a..3436809 100644
--- a/harry-integration/test/harry/model/QuerySelectorTest.java
+++ b/harry-integration/test/harry/model/QuerySelectorTest.java
@@ -31,11 +31,11 @@ import harry.ddl.SchemaGenerators;
import harry.ddl.SchemaSpec;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
-import harry.runner.MutatingPartitionVisitor;
-import harry.runner.MutatingRowVisitor;
-import harry.runner.PartitionVisitor;
-import harry.runner.Query;
-import harry.runner.QueryGenerator;
+import harry.visitors.MutatingPartitionVisitor;
+import harry.visitors.MutatingRowVisitor;
+import harry.visitors.PartitionVisitor;
+import harry.operations.Query;
+import harry.operations.QueryGenerator;
import static harry.generators.DataGenerators.NIL_DESCR;
diff --git a/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java b/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
index 7d5956f..55fb00b 100644
--- a/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
+++ b/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
@@ -29,9 +29,9 @@ import harry.corruptor.HideValueCorruptor;
import harry.corruptor.QueryResponseCorruptor;
import harry.corruptor.QueryResponseCorruptor.SimpleQueryResponseCorruptor;
import harry.ddl.SchemaSpec;
-import harry.runner.PartitionVisitor;
-import harry.runner.Query;
-import harry.runner.SinglePartitionValidator;
+import harry.visitors.PartitionVisitor;
+import harry.operations.Query;
+import harry.visitors.SinglePartitionValidator;
public class QuiescentCheckerIntegrationTest extends ModelTestBase
{
diff --git a/harry-integration/test/harry/model/TestEveryClustering.java b/harry-integration/test/harry/model/TestEveryClustering.java
index f844b41..f3a2ba8 100644
--- a/harry-integration/test/harry/model/TestEveryClustering.java
+++ b/harry-integration/test/harry/model/TestEveryClustering.java
@@ -1,28 +1,22 @@
package harry.model;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.function.Supplier;
-
-import org.junit.Assert;
-import org.junit.Test;
-
import harry.core.Configuration;
import harry.core.Run;
import harry.ddl.SchemaGenerators;
import harry.ddl.SchemaSpec;
import harry.generators.distribution.Distribution;
-import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
+import harry.operations.Query;
import harry.operations.Relation;
-import harry.runner.FaultInjectingPartitionVisitor;
-import harry.runner.LoggingPartitionVisitor;
-import harry.runner.MutatingPartitionVisitor;
-import harry.runner.MutatingRowVisitor;
-import harry.runner.PartitionVisitor;
-import harry.runner.Query;
-import harry.runner.QueryGenerator;
+import harry.visitors.LoggingPartitionVisitor;
+import harry.visitors.MutatingRowVisitor;
+import harry.visitors.PartitionVisitor;
import org.apache.cassandra.distributed.api.IInvokableInstance;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.function.Supplier;
public class TestEveryClustering extends IntegrationTestBase
{
diff --git a/harry-integration/test/harry/op/RowVisitorTest.java b/harry-integration/test/harry/op/RowVisitorTest.java
index ae51115..a644afa 100644
--- a/harry-integration/test/harry/op/RowVisitorTest.java
+++ b/harry-integration/test/harry/op/RowVisitorTest.java
@@ -35,7 +35,7 @@ import harry.model.clock.OffsetClock;
import harry.model.sut.SystemUnderTest;
import harry.operations.CompiledStatement;
import harry.runner.DataTracker;
-import harry.runner.MutatingRowVisitor;
+import harry.visitors.MutatingRowVisitor;
import org.apache.cassandra.cql3.CQLTester;
import static harry.model.OpSelectors.DefaultDescriptorSelector.DEFAULT_OP_SELECTOR;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org
[cassandra-harry] 02/05: Integration improvements
Posted by if...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ifesdjeen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-harry.git
commit 469533e2944fc12d4c82f13736160dd2eb143d44
Author: Alex Petrov <ol...@gmail.com>
AuthorDate: Mon Jul 12 17:54:42 2021 +0200
Integration improvements
---
.../test/resources/single_partition_test.yml | 55 ++++++
harry-integration-external/pom.xml | 5 +
.../model/sut/external/ExternalClusterSut.java | 14 +-
harry-integration/pom.xml | 14 +-
.../src/harry/model/sut/ExternalClusterSut.java | 187 ---------------------
.../src/harry/model/sut/InJvmSutBase.java | 3 +-
.../src/harry/runner/HarryRunnerJvm.java | 43 +++++
harry-integration/test/conf/cassandra.yaml | 44 +++++
.../test/harry/ddl/SchemaGenTest.java | 15 +-
.../generators/DataGeneratorsIntegrationTest.java | 104 ++++++++++++
.../test/harry/model/IntegrationTestBase.java | 6 +-
.../model/QuiescentCheckerIntegrationTest.java | 3 +-
.../test/harry/model/TestEveryClustering.java | 89 ++++++++++
.../test/harry/op/RowVisitorTest.java | 8 +-
.../test/resources/single_partition_test.yml | 55 ++++++
15 files changed, 435 insertions(+), 210 deletions(-)
diff --git a/harry-integration-backup/test/resources/single_partition_test.yml b/harry-integration-backup/test/resources/single_partition_test.yml
new file mode 100644
index 0000000..0ebe2aa
--- /dev/null
+++ b/harry-integration-backup/test/resources/single_partition_test.yml
@@ -0,0 +1,55 @@
+seed: 1
+
+# Default schema provider generates random schema
+schema_provider:
+ default: {}
+
+drop_schema: false
+create_schema: true
+truncate_table: false
+
+clock:
+ offset:
+ offset: 1000
+
+run_time: 10
+run_time_unit: "MINUTES"
+
+system_under_test:
+ println: {}
+
+partition_descriptor_selector:
+ always_same:
+ pd: 12345
+
+clustering_descriptor_selector:
+ default:
+ modifications_per_lts:
+ type: "constant"
+ constant: 2
+ rows_per_modification:
+ type: "constant"
+ constant: 2
+ operation_kind_weights:
+ DELETE_RANGE: 1
+ DELETE_SLICE: 1
+ DELETE_ROW: 1
+ DELETE_COLUMN: 1
+ DELETE_PARTITION: 1
+ DELETE_COLUMN_WITH_STATICS: 1
+ INSERT_WITH_STATICS: 24
+ INSERT: 24
+ UPDATE_WITH_STATICS: 23
+ UPDATE: 23
+ column_mask_bitsets: null
+ max_partition_size: 100
+
+data_tracker:
+ no_op: {}
+
+runner:
+ sequential:
+ partition_visitors: []
+
+metric_reporter:
+ no_op: {}
\ No newline at end of file
diff --git a/harry-integration-external/pom.xml b/harry-integration-external/pom.xml
index 8c57e0e..0b49f6d 100755
--- a/harry-integration-external/pom.xml
+++ b/harry-integration-external/pom.xml
@@ -34,6 +34,11 @@
<dependencies>
<dependency>
+ <groupId>com.datastax.cassandra</groupId>
+ <artifactId>cassandra-driver-core</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.cassandra</groupId>
<artifactId>harry-core</artifactId>
<version>${project.parent.version}</version>
diff --git a/harry-integration-external/src/harry/model/sut/external/ExternalClusterSut.java b/harry-integration-external/src/harry/model/sut/external/ExternalClusterSut.java
index e471ede..76a4e3c 100644
--- a/harry-integration-external/src/harry/model/sut/external/ExternalClusterSut.java
+++ b/harry-integration-external/src/harry/model/sut/external/ExternalClusterSut.java
@@ -93,9 +93,7 @@ public class ExternalClusterSut implements SystemUnderTest
{
try
{
- Statement st = new SimpleStatement(statement, bindings);
- st.setConsistencyLevel(toDriverCl(cl));
- return resultSetToObjectArray(session.execute(st));
+ return resultSetToObjectArray(session.execute(statement, bindings));
}
catch (Throwable t)
{
@@ -134,9 +132,7 @@ public class ExternalClusterSut implements SystemUnderTest
public CompletableFuture<Object[][]> executeAsync(String statement, ConsistencyLevel cl, Object... bindings)
{
CompletableFuture<Object[][]> future = new CompletableFuture<>();
- Statement st = new SimpleStatement(statement, bindings);
- st.setConsistencyLevel(toDriverCl(cl));
- Futures.addCallback(session.executeAsync(st),
+ Futures.addCallback(session.executeAsync(statement, bindings),
new FutureCallback<ResultSet>()
{
public void onSuccess(ResultSet rows)
@@ -185,8 +181,10 @@ public class ExternalClusterSut implements SystemUnderTest
{
switch (cl)
{
- case ALL: return com.datastax.driver.core.ConsistencyLevel.ALL;
- case QUORUM: return com.datastax.driver.core.ConsistencyLevel.QUORUM;
+ case ALL:
+ return com.datastax.driver.core.ConsistencyLevel.ALL;
+ case QUORUM:
+ return com.datastax.driver.core.ConsistencyLevel.QUORUM;
}
throw new IllegalArgumentException("Don't know a CL: " + cl);
}
diff --git a/harry-integration/pom.xml b/harry-integration/pom.xml
index 4695d38..0b2d131 100755
--- a/harry-integration/pom.xml
+++ b/harry-integration/pom.xml
@@ -34,9 +34,14 @@
<dependencies>
<dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.cassandra</groupId>
<artifactId>harry-core</artifactId>
- <version>${project.parent.version}</version>
+ <version>${project.parent.version}</version>
</dependency>
<dependency>
@@ -62,5 +67,12 @@
<scope>test</scope>
</dependency>
</dependencies>
+ <build>
+ <testResources>
+ <testResource>
+ <directory>test/resources</directory>
+ </testResource>
+ </testResources>
+ </build>
</project>
diff --git a/harry-integration/src/harry/model/sut/ExternalClusterSut.java b/harry-integration/src/harry/model/sut/ExternalClusterSut.java
deleted file mode 100644
index 74085e4..0000000
--- a/harry-integration/src/harry/model/sut/ExternalClusterSut.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package harry.model.sut;
-
-import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import com.google.common.util.concurrent.FutureCallback;
-import com.google.common.util.concurrent.Futures;
-
-import com.datastax.driver.core.Cluster;
-import com.datastax.driver.core.ColumnDefinitions;
-import com.datastax.driver.core.QueryOptions;
-import com.datastax.driver.core.ResultSet;
-import com.datastax.driver.core.Row;
-import com.datastax.driver.core.Session;
-import com.datastax.driver.core.SimpleStatement;
-import com.datastax.driver.core.Statement;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import harry.core.Configuration;
-
-public class ExternalClusterSut implements SystemUnderTest
-{
- public static void init()
- {
- Configuration.registerSubtypes(ExternalClusterSutConfiguration.class);
- }
-
- private final Session session;
- private final ExecutorService executor;
-
- // TODO: pass cluster, not session
- public ExternalClusterSut(Session session)
- {
- this(session, 10);
- }
-
- public ExternalClusterSut(Session session, int threads)
- {
- this.session = session;
- this.executor = Executors.newFixedThreadPool(threads);
- }
-
- public static ExternalClusterSut create()
- {
- return new ExternalClusterSut(Cluster.builder()
- .withQueryOptions(new QueryOptions().setConsistencyLevel(toDriverCl(ConsistencyLevel.QUORUM)))
- .addContactPoints("127.0.0.1")
- .build()
- .connect());
- }
-
- public boolean isShutdown()
- {
- return session.isClosed();
- }
-
- public void shutdown()
- {
- session.close();
- executor.shutdown();
- try
- {
- executor.awaitTermination(60, TimeUnit.SECONDS);
- }
- catch (InterruptedException e)
- {
- throw new RuntimeException(e);
- }
- }
-
- // TODO: this is rather simplistic
- public Object[][] execute(String statement, ConsistencyLevel cl, Object... bindings)
- {
- int repeat = 10;
- while (true)
- {
- try
- {
- Statement st = new SimpleStatement(statement, bindings);
- st.setConsistencyLevel(toDriverCl(cl));
- return resultSetToObjectArray(session.execute(st));
- }
- catch (Throwable t)
- {
- if (repeat < 0)
- throw t;
-
- t.printStackTrace();
- repeat--;
- // retry unconditionally
- }
- }
- }
-
- public static Object[][] resultSetToObjectArray(ResultSet rs)
- {
- List<Row> rows = rs.all();
- if (rows.size() == 0)
- return new Object[0][];
- Object[][] results = new Object[rows.size()][];
- for (int i = 0; i < results.length; i++)
- {
- Row row = rows.get(i);
- ColumnDefinitions cds = row.getColumnDefinitions();
- Object[] result = new Object[cds.size()];
- for (int j = 0; j < cds.size(); j++)
- {
- if (!row.isNull(j))
- result[j] = row.getObject(j);
- }
- results[i] = result;
- }
- return results;
- }
-
- public CompletableFuture<Object[][]> executeAsync(String statement, ConsistencyLevel cl, Object... bindings)
- {
- CompletableFuture<Object[][]> future = new CompletableFuture<>();
- Statement st = new SimpleStatement(statement, bindings);
- st.setConsistencyLevel(toDriverCl(cl));
- Futures.addCallback(session.executeAsync(st),
- new FutureCallback<ResultSet>()
- {
- public void onSuccess(ResultSet rows)
- {
- future.complete(resultSetToObjectArray(rows));
- }
-
- public void onFailure(Throwable throwable)
- {
- future.completeExceptionally(throwable);
- }
- },
- executor);
-
- return future;
- }
-
- public static com.datastax.driver.core.ConsistencyLevel toDriverCl(SystemUnderTest.ConsistencyLevel cl)
- {
- switch (cl)
- {
- case ALL: return com.datastax.driver.core.ConsistencyLevel.ALL;
- case QUORUM: return com.datastax.driver.core.ConsistencyLevel.QUORUM;
- }
- throw new IllegalArgumentException("Don't know a CL: " + cl);
- }
-
- @JsonTypeName("external")
- public static class ExternalClusterSutConfiguration implements Configuration.SutConfiguration
- {
- public final String[] hosts;
-
- public ExternalClusterSutConfiguration(@JsonProperty(value = "hosts") String[] hosts)
- {
- this.hosts = hosts;
- }
-
- public SystemUnderTest make()
- {
- Cluster cluster = Cluster.builder().addContactPoints(hosts).build();
- Session session = cluster.newSession().init();
- return new ExternalClusterSut(session);
- }
- }
-}
\ No newline at end of file
diff --git a/harry-integration/src/harry/model/sut/InJvmSutBase.java b/harry-integration/src/harry/model/sut/InJvmSutBase.java
index e29481f..aa2608f 100644
--- a/harry-integration/src/harry/model/sut/InJvmSutBase.java
+++ b/harry-integration/src/harry/model/sut/InJvmSutBase.java
@@ -30,20 +30,19 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
-import com.google.common.collect.Iterators;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import harry.core.Configuration;
-import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.distributed.api.ICluster;
import org.apache.cassandra.distributed.api.IInstance;
import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.distributed.api.IMessage;
import org.apache.cassandra.distributed.api.IMessageFilters;
+import relocated.shaded.com.google.common.collect.Iterators;
public class InJvmSutBase<NODE extends IInstance, CLUSTER extends ICluster<NODE>> implements SystemUnderTest.FaultInjectingSut
{
diff --git a/harry-integration/src/harry/runner/HarryRunnerJvm.java b/harry-integration/src/harry/runner/HarryRunnerJvm.java
new file mode 100644
index 0000000..fbb30b9
--- /dev/null
+++ b/harry-integration/src/harry/runner/HarryRunnerJvm.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package harry.runner;
+
+import harry.core.Configuration;
+import harry.model.sut.InJvmSut;
+
+import java.io.File;
+
+public class HarryRunnerJvm extends HarryRunner {
+
+ public static void main(String[] args) throws Throwable {
+ InJvmSut.init();
+
+ HarryRunnerJvm runner = new HarryRunnerJvm();
+ File configFile = runner.loadConfig(args);
+
+ Configuration configuration = Configuration.fromFile(configFile);
+ runner.run(configuration);
+ }
+
+
+ @Override
+ public void beforeRun(Runner runner) {
+
+ }
+}
diff --git a/harry-integration/test/conf/cassandra.yaml b/harry-integration/test/conf/cassandra.yaml
new file mode 100644
index 0000000..2536aa8
--- /dev/null
+++ b/harry-integration/test/conf/cassandra.yaml
@@ -0,0 +1,44 @@
+#
+# Warning!
+# Consider the effects on 'o.a.c.i.s.LegacySSTableTest' before changing schemas in this file.
+#
+cluster_name: Test Cluster
+memtable_allocation_type: heap_buffers
+commitlog_sync: batch
+commitlog_sync_batch_window_in_ms: 1.0
+commitlog_segment_size_in_mb: 5
+commitlog_directory: build/test/cassandra/commitlog
+hints_directory: build/test/cassandra/hints
+partitioner: org.apache.cassandra.dht.ByteOrderedPartitioner
+listen_address: 127.0.0.1
+storage_port: 7010
+rpc_port: 9170
+start_native_transport: true
+native_transport_port: 9042
+column_index_size_in_kb: 4
+saved_caches_directory: build/test/cassandra/saved_caches
+data_file_directories:
+ - build/test/cassandra/data
+disk_access_mode: mmap
+seed_provider:
+ - class_name: org.apache.cassandra.locator.SimpleSeedProvider
+ parameters:
+ - seeds: "127.0.0.1"
+endpoint_snitch: org.apache.cassandra.locator.SimpleSnitch
+dynamic_snitch: true
+request_scheduler: org.apache.cassandra.scheduler.RoundRobinScheduler
+request_scheduler_id: keyspace
+server_encryption_options:
+ internode_encryption: none
+ keystore: conf/.keystore
+ keystore_password: cassandra
+ truststore: conf/.truststore
+ truststore_password: cassandra
+incremental_backups: true
+concurrent_compactors: 4
+compaction_throughput_mb_per_sec: 0
+row_cache_class_name: org.apache.cassandra.cache.OHCProvider
+row_cache_size_in_mb: 16
+enable_user_defined_functions: true
+enable_scripted_user_defined_functions: true
+enable_drop_compact_storage: true
diff --git a/harry-integration/test/harry/ddl/SchemaGenTest.java b/harry-integration/test/harry/ddl/SchemaGenTest.java
index 74a9c53..2f7e898 100644
--- a/harry-integration/test/harry/ddl/SchemaGenTest.java
+++ b/harry-integration/test/harry/ddl/SchemaGenTest.java
@@ -50,11 +50,12 @@ public class SchemaGenTest extends CQLTester
// TODO: compact storage tests
@Test
- public void testSelectForwardAndReverseIteration()
+ public void testSelectForwardAndReverseIteration() throws Throwable
{
Generator<SchemaSpec> gen = new SchemaGenerators.Builder(KEYSPACE).partitionKeyColumnCount(1, 4)
.clusteringColumnCount(1, 10)
.regularColumnCount(0, 10)
+ .staticColumnCount(0, 10)
.generator();
@@ -78,11 +79,12 @@ public class SchemaGenTest extends CQLTester
}
@Test
- public void createTableRoundTrip()
+ public void createTableRoundTrip() throws Throwable
{
Generator<SchemaSpec> gen = new SchemaGenerators.Builder(KEYSPACE).partitionKeyColumnCount(1, 10)
- .clusteringColumnCount(0, 10)
+ .clusteringColumnCount(1, 10)
.regularColumnCount(0, 10)
+ .staticColumnCount(0, 10)
.generator();
TestRunner.test(gen,
@@ -93,6 +95,7 @@ public class SchemaGenTest extends CQLTester
compareColumns(schemaDefinition.partitionKeys, tableMetadata.partitionKeyColumns());
compareColumns(schemaDefinition.clusteringKeys, tableMetadata.clusteringColumns());
compareColumns(schemaDefinition.regularColumns, tableMetadata.regularColumns());
+ compareColumns(schemaDefinition.staticColumns, tableMetadata.staticColumns());
});
}
@@ -108,8 +111,8 @@ public class SchemaGenTest extends CQLTester
ColumnSpec.regularColumn("v2", ColumnSpec.asciiType),
ColumnSpec.regularColumn("v3", ColumnSpec.int64Type),
ColumnSpec.regularColumn("v4", ColumnSpec.int64Type)),
- Arrays.asList(ColumnSpec.staticColumn("regular1", ColumnSpec.asciiType),
- ColumnSpec.staticColumn("regular2", ColumnSpec.int64Type)));
+ Arrays.asList(ColumnSpec.staticColumn("static1", ColumnSpec.asciiType),
+ ColumnSpec.staticColumn("static2", ColumnSpec.int64Type)));
String tableDef = spec.compile().cql();
@@ -118,6 +121,7 @@ public class SchemaGenTest extends CQLTester
compareColumns(spec.partitionKeys, tableMetadata.partitionKeyColumns());
compareColumns(spec.clusteringKeys, tableMetadata.clusteringColumns());
compareColumns(spec.regularColumns, tableMetadata.regularColumns());
+ compareColumns(spec.staticColumns, tableMetadata.staticColumns());
}
@@ -126,6 +130,7 @@ public class SchemaGenTest extends CQLTester
{
Gen<Pair<Integer, Integer>> ckCounts = integers().between(0, 4).zip(integers().between(0, 6), Pair::create);
Gen<Pair<Integer, Integer>> regCounts = integers().between(0, 4).zip(integers().between(0, 6), Pair::create);
+// Gen<Pair<Integer, Integer>> staticCounts = integers().between(0, 4).zip(integers().between(0, 6), Pair::create);
Gen<Pair<Integer, Integer>> pkCounts = integers().between(1, 4).zip(integers().between(0, 6), Pair::create);
Gen<SchemaGenerationInputs> inputs = pkCounts.zip(ckCounts, regCounts,
diff --git a/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java b/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
index affd954..cadee0c 100644
--- a/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
+++ b/harry-integration/test/harry/generators/DataGeneratorsIntegrationTest.java
@@ -19,11 +19,28 @@
package harry.generators;
import java.util.Random;
+import java.util.concurrent.CompletableFuture;
import org.junit.Test;
+import harry.core.Configuration;
+import harry.core.Run;
import harry.ddl.ColumnSpec;
+import harry.ddl.SchemaGenerators;
+import harry.ddl.SchemaSpec;
+import harry.generators.distribution.Distribution;
+import harry.model.NoOpChecker;
+import harry.model.OpSelectors;
+import harry.model.sut.SystemUnderTest;
+import harry.runner.MutatingPartitionVisitor;
+import harry.runner.MutatingRowVisitor;
+import harry.runner.PartitionVisitor;
+import harry.runner.SinglePartitionValidator;
+import harry.util.TestRunner;
import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.distributed.impl.RowUtil;
+import relocated.shaded.com.google.common.collect.Iterators;
public class DataGeneratorsIntegrationTest extends CQLTester
{
@@ -60,4 +77,91 @@ public class DataGeneratorsIntegrationTest extends CQLTester
}
}
}
+
+ @Test
+ public void queryParseabilityTest() throws Throwable
+ {
+ Generator<SchemaSpec> gen = new SchemaGenerators.Builder(KEYSPACE).partitionKeyColumnCount(2, 4)
+ .clusteringColumnCount(1, 4)
+ .regularColumnCount(1, 4)
+ .staticColumnCount(1, 4)
+ .generator();
+
+ TestRunner.test(gen,
+ (schema) -> {
+ createTable(schema.compile().cql());
+
+ Configuration.ConfigurationBuilder builder = Configuration.fromFile(getClass().getClassLoader().getResource("single_partition_test.yml").getFile())
+ .unbuild()
+ .setSchemaProvider(new Configuration.FixedSchemaProviderConfiguration(schema, null, null, null, null))
+ .setSUT(CqlTesterSut::new);
+
+ for (OpSelectors.OperationKind opKind : OpSelectors.OperationKind.values())
+ {
+ Run run = builder
+ .setClusteringDescriptorSelector((rng, schema_) -> {
+ return new OpSelectors.DefaultDescriptorSelector(rng,
+ OpSelectors.columnSelectorBuilder().forAll(schema_).build(),
+ OpSelectors.OperationSelector.weighted(Surjections.weights(100), opKind),
+ new Distribution.ConstantDistribution(2),
+ new Distribution.ConstantDistribution(2),
+ 100);
+ })
+ .build()
+ .createRun();
+
+ PartitionVisitor visitor = new MutatingPartitionVisitor(run, MutatingRowVisitor::new);
+ for (int lts = 0; lts < 100; lts++)
+ visitor.visitPartition(lts);
+ }
+
+ Run run = builder.build()
+ .createRun();
+ PartitionVisitor visitor = new SinglePartitionValidator(100, run, NoOpChecker::new);
+ for (int lts = 0; lts < 100; lts++)
+ visitor.visitPartition(lts);
+
+ });
+
+ }
+
+ public class CqlTesterSut implements SystemUnderTest
+ {
+ public boolean isShutdown()
+ {
+ return false;
+ }
+
+ public void shutdown()
+ {
+ cleanup();
+ }
+
+ public void schemaChange(String statement)
+ {
+ createTable(statement);
+ }
+
+ public Object[][] execute(String statement, ConsistencyLevel cl, Object... bindings)
+ {
+ try
+ {
+ UntypedResultSet res = DataGeneratorsIntegrationTest.this.execute(statement, bindings);
+ if (res == null)
+ return new Object[][] {};
+
+ return Iterators.toArray(RowUtil.toIter(res), Object[].class);
+ }
+ catch (Throwable throwable)
+ {
+ throw new RuntimeException(throwable);
+ }
+ }
+
+ public CompletableFuture<Object[][]> executeAsync(String statement, ConsistencyLevel cl, Object... bindings)
+ {
+ return CompletableFuture.completedFuture(execute(statement, cl, bindings));
+ }
+ }
}
+
diff --git a/harry-integration/test/harry/model/IntegrationTestBase.java b/harry-integration/test/harry/model/IntegrationTestBase.java
index 551124f..a6a9697 100644
--- a/harry-integration/test/harry/model/IntegrationTestBase.java
+++ b/harry-integration/test/harry/model/IntegrationTestBase.java
@@ -81,8 +81,10 @@ public class IntegrationTestBase extends TestBaseImpl
.addWeight(OpSelectors.OperationKind.DELETE_SLICE, 1)
.addWeight(OpSelectors.OperationKind.DELETE_PARTITION, 1)
.addWeight(OpSelectors.OperationKind.DELETE_COLUMN_WITH_STATICS, 5)
- .addWeight(OpSelectors.OperationKind.WRITE_WITH_STATICS, 45)
- .addWeight(OpSelectors.OperationKind.WRITE, 45)
+ .addWeight(OpSelectors.OperationKind.INSERT_WITH_STATICS, 20)
+ .addWeight(OpSelectors.OperationKind.INSERT, 20)
+ .addWeight(OpSelectors.OperationKind.UPDATE_WITH_STATICS, 25)
+ .addWeight(OpSelectors.OperationKind.UPDATE, 25)
.build());
}
diff --git a/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java b/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
index 4c83276..7d5956f 100644
--- a/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
+++ b/harry-integration/test/harry/model/QuiescentCheckerIntegrationTest.java
@@ -128,7 +128,8 @@ public class QuiescentCheckerIntegrationTest extends ModelTestBase
(t, run) -> {
String expected = "doesn't match the one predicted by the model";
String expected2 = "don't match ones predicted by the model";
- if (t.getMessage().contains(expected) || t.getMessage().contains(expected2))
+ String expected3 = "Found a row in the model that is not present in the resultset";
+ if (t.getMessage().contains(expected) || t.getMessage().contains(expected2) || t.getMessage().contains(expected3))
return;
throw new AssertionError(String.format("Exception string mismatch.\nExpected error: %s.\nActual error: %s", expected, t.getMessage()),
diff --git a/harry-integration/test/harry/model/TestEveryClustering.java b/harry-integration/test/harry/model/TestEveryClustering.java
new file mode 100644
index 0000000..f844b41
--- /dev/null
+++ b/harry-integration/test/harry/model/TestEveryClustering.java
@@ -0,0 +1,89 @@
+package harry.model;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.function.Supplier;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import harry.core.Configuration;
+import harry.core.Run;
+import harry.ddl.SchemaGenerators;
+import harry.ddl.SchemaSpec;
+import harry.generators.distribution.Distribution;
+import harry.model.sut.SystemUnderTest;
+import harry.operations.CompiledStatement;
+import harry.operations.Relation;
+import harry.runner.FaultInjectingPartitionVisitor;
+import harry.runner.LoggingPartitionVisitor;
+import harry.runner.MutatingPartitionVisitor;
+import harry.runner.MutatingRowVisitor;
+import harry.runner.PartitionVisitor;
+import harry.runner.Query;
+import harry.runner.QueryGenerator;
+import org.apache.cassandra.distributed.api.IInvokableInstance;
+
+public class TestEveryClustering extends IntegrationTestBase
+{
+ int CYCLES = 1000;
+
+ @Test
+ public void basicQuerySelectorTest()
+ {
+ Supplier<SchemaSpec> schemaGen = SchemaGenerators.progression(SchemaGenerators.DEFAULT_SWITCH_AFTER);
+ for (int cnt = 0; cnt < Integer.MAX_VALUE; cnt++)
+ {
+ beforeEach();
+ SchemaSpec schemaSpec = schemaGen.get();
+
+ System.out.println(schemaSpec.compile().cql());
+ int partitionSize = 1000;
+
+ Configuration config = sharedConfiguration(cnt, schemaSpec)
+ .setPartitionDescriptorSelector(new Configuration.DefaultPDSelectorConfiguration(1, partitionSize))
+ .setClusteringDescriptorSelector(sharedCDSelectorConfiguration()
+ .setNumberOfModificationsDistribution(() -> new Distribution.ConstantDistribution(1L))
+ .setRowsPerModificationDistribution(() -> new Distribution.ConstantDistribution(1L))
+ .setMaxPartitionSize(250)
+ .build())
+ .build();
+
+ Run run = config.createRun();
+ run.sut.schemaChange(run.schemaSpec.compile().cql());
+ OpSelectors.MonotonicClock clock = run.clock;
+
+ Set<Long> visitedCds = new HashSet<>();
+ PartitionVisitor partitionVisitor = new LoggingPartitionVisitor(run, (r) -> {
+ return new MutatingRowVisitor(r) {
+ public CompiledStatement perform(OpSelectors.OperationKind op, long lts, long pd, long cd, long opId)
+ {
+ visitedCds.add(cd);
+ return super.perform(op, lts, pd, cd, opId);
+ }
+ };
+ });
+ sut.cluster().stream().forEach((IInvokableInstance node) -> node.nodetool("disableautocompaction"));
+ for (int i = 0; i < CYCLES; i++)
+ {
+ long lts = clock.nextLts();
+ partitionVisitor.visitPartition(lts);
+
+ if (i > 0 && i % 250 == 0)
+ sut.cluster().stream().forEach((IInvokableInstance node) -> node.nodetool("flush", schemaSpec.keyspace, schemaSpec.table));
+ }
+
+ for (Long cd : visitedCds)
+ {
+ Query query = new Query.SingleClusteringQuery(Query.QueryKind.SINGLE_CLUSTERING,
+ run.pdSelector.pd(0),
+ cd,
+ false,
+ Relation.eqRelations(run.schemaSpec.ckGenerator.slice(cd), run.schemaSpec.clusteringKeys),
+ run.schemaSpec);
+ Model model = new QuiescentChecker(run);
+ model.validate(query);
+ }
+ }
+ }
+}
diff --git a/harry-integration/test/harry/op/RowVisitorTest.java b/harry-integration/test/harry/op/RowVisitorTest.java
index d7061f9..ae51115 100644
--- a/harry-integration/test/harry/op/RowVisitorTest.java
+++ b/harry-integration/test/harry/op/RowVisitorTest.java
@@ -83,10 +83,10 @@ public class RowVisitorTest extends CQLTester
MutatingRowVisitor visitor = new MutatingRowVisitor(run);
long[] descriptors = rand.next(4);
- execute(visitor.write(Math.abs(descriptors[0]),
- descriptors[1],
- descriptors[2],
- descriptors[3]));
+ execute(visitor.insert(Math.abs(descriptors[0]),
+ descriptors[1],
+ descriptors[2],
+ descriptors[3]));
}
}
diff --git a/harry-integration/test/resources/single_partition_test.yml b/harry-integration/test/resources/single_partition_test.yml
new file mode 100644
index 0000000..0ebe2aa
--- /dev/null
+++ b/harry-integration/test/resources/single_partition_test.yml
@@ -0,0 +1,55 @@
+seed: 1
+
+# Default schema provider generates random schema
+schema_provider:
+ default: {}
+
+drop_schema: false
+create_schema: true
+truncate_table: false
+
+clock:
+ offset:
+ offset: 1000
+
+run_time: 10
+run_time_unit: "MINUTES"
+
+system_under_test:
+ println: {}
+
+partition_descriptor_selector:
+ always_same:
+ pd: 12345
+
+clustering_descriptor_selector:
+ default:
+ modifications_per_lts:
+ type: "constant"
+ constant: 2
+ rows_per_modification:
+ type: "constant"
+ constant: 2
+ operation_kind_weights:
+ DELETE_RANGE: 1
+ DELETE_SLICE: 1
+ DELETE_ROW: 1
+ DELETE_COLUMN: 1
+ DELETE_PARTITION: 1
+ DELETE_COLUMN_WITH_STATICS: 1
+ INSERT_WITH_STATICS: 24
+ INSERT: 24
+ UPDATE_WITH_STATICS: 23
+ UPDATE: 23
+ column_mask_bitsets: null
+ max_partition_size: 100
+
+data_tracker:
+ no_op: {}
+
+runner:
+ sequential:
+ partition_visitors: []
+
+metric_reporter:
+ no_op: {}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org