You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by yc...@apache.org on 2023/10/06 20:00:34 UTC
[cassandra-analytics] branch trunk updated: CASSANDRA-18759: Use in-jvm dtest framework from Sidecar for testing
This is an automated email from the ASF dual-hosted git repository.
ycai pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-analytics.git
The following commit(s) were added to refs/heads/trunk by this push:
new deebdf9 CASSANDRA-18759: Use in-jvm dtest framework from Sidecar for testing
deebdf9 is described below
commit deebdf97ad01f23550d7d3b42d98c7bf111e2f95
Author: Doug Rohrer <dr...@apple.com>
AuthorDate: Wed Jun 14 13:33:29 2023 -0400
CASSANDRA-18759: Use in-jvm dtest framework from Sidecar for testing
This commit introduces the use of the in-jvm dtest framework for testing
Analytics workloads. It can spin up a Cassandra cluster, including the
necessary Sidecar process, to test writing to and reading from Cassandra
using the analytics library.
Additional changes made in this commit include
* Use concurrent collections in MockBulkWriterContext (Fixes flaky test StreamSessionConsistencyTest)
The StreamSessionConsistency test uses the MockBulkWriter context, but it wasn't originally used
(before this test was added) in a multi-threaded environment. Because of this, it would occasionally
throw ConcurrentModificationExceptions, which would cause the stream test to fail in a
non-deterministic way. This commit adds the use of concurrent/synchronous collections to the
MockBulkWriterContext to make sure it doesn't throw these spurious errors.
* Make the StartupValidation system thread-safe by using TreadLocals
instead of static collections, and clearing them once validation is
complete.
Patch by Doug Rohrer; Reviewed by Dinesh Joshi, Francisco Guerrero, Yifan Cai for CASSANDRA-18759
---
.circleci/config.yml | 77 ++++-
.gitignore | 7 +
CHANGES.txt | 1 +
DEV-README.md | 34 ++-
README.md | 4 +-
build.gradle | 69 ++++-
cassandra-analytics-core-example/README.md | 20 +-
cassandra-analytics-core-example/build.gradle | 4 +-
cassandra-analytics-core/build.gradle | 38 +--
.../spark/validation/KeyStoreValidation.java | 7 +-
.../spark/validation/StartupValidator.java | 29 +-
.../spark/bulkwriter/MockBulkWriterContext.java | 10 +-
.../build.gradle | 75 +++++
.../testing/CassandraSidecarTestContext.java | 252 +++++++++++++++++
.../sidecar/testing/IntegrationTestBase.java | 234 +++++++++++++++
.../sidecar/testing/IntegrationTestModule.java | 105 +++++++
.../testing/AbstractCassandraTestContext.java | 83 ++++++
.../testing/CassandraIntegrationTest.java | 127 +++++++++
.../cassandra/testing/CassandraTestContext.java | 51 ++++
.../cassandra/testing/CassandraTestTemplate.java | 314 +++++++++++++++++++++
.../testing/ConfigurableCassandraTestContext.java | 73 +++++
.../cassandra/testing/SimpleCassandraVersion.java | 169 +++++++++++
.../org/apache/cassandra/testing/TestVersion.java | 37 +++
.../cassandra/testing/TestVersionSupplier.java | 47 +++
cassandra-analytics-integration-tests/build.gradle | 88 ++++++
.../cassandra/analytics/IntegrationTestJob.java | 301 ++++++++++++++++++++
.../analytics/SparkBulkWriterSimpleTest.java | 46 +++
.../src/test/resources/logback-test.xml | 48 ++++
.../apache/cassandra/bridge/CassandraVersion.java | 1 -
cassandra-four-zero/build.gradle | 15 +-
config/checkstyle/checkstyle.xml | 13 +-
gradle/wrapper/gradle-wrapper.jar | Bin 58702 -> 59536 bytes
gradle/wrapper/gradle-wrapper.properties | 19 +-
gradlew | 273 +++++++++++-------
profiles/scala-2.11-spark-2-jdk-1.8.gradle | 5 +-
profiles/scala-2.12-spark-2-jdk-1.8.gradle | 5 +-
profiles/scala-2.12-spark-3-jdk-11.gradle | 3 +-
profiles/scala-2.13-spark-3-jdk-11.gradle | 3 +-
scripts/.mvn/wrapper/maven-wrapper.jar | Bin 0 -> 62547 bytes
.../.mvn/wrapper/maven-wrapper.properties | 8 +-
scripts/build-dependencies.sh | 23 ++
scripts/build-dtest-jars.sh | 121 ++++++++
scripts/build-shaded-dtest-jar-local.sh | 58 ++++
scripts/build-sidecar.sh | 75 +++++
scripts/functions.sh | 42 +++
scripts/mvnw | 308 ++++++++++++++++++++
scripts/relocate-dtest-dependencies.pom | 192 +++++++++++++
settings.gradle | 2 +
48 files changed, 3295 insertions(+), 221 deletions(-)
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 4b9da6f..df4b720 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -16,18 +16,43 @@
# under the License.
version: 2.1
+commands:
+ install_common:
+ steps:
+ - run: sudo apt-get update
+ # Somehow, the certs alreeady installed in the java 11 image are installed strangely
+ # in that there's a directory called `cacerts` with a symlink in it named `cacerts`
+ # (/etc/ssl/certs/java/cacerts/cacerts) rather than just a symlink to
+ # `cacerts` in the /etc/ssl/certs/java directory - if this is the case, fix it by
+ # moving it down a directory and then ant should install cleanly
+ - run: |
+ if [ -f /etc/ssl/certs/java/cacerts/cacerts ]; then
+ sudo mv /etc/ssl/certs/java/cacerts/ /etc/ssl/certs/java/cacerts-old
+ sudo mv /etc/ssl/certs/java/cacerts-old/cacerts /etc/ssl/certs/java/
+ sudo rmdir /etc/ssl/certs/java/cacerts-old
+ fi
+ # The image already has java installed, but `apt-get install ant` reinstalls it.
+ # Therefore, we download just the deb package for ant and install it with dpkg.
+ - run: |
+ apt-get download ant ant-optional
+ sudo dpkg --force-all -i ant*.deb
+ rm ant*.deb
+ # We need aliases for localhost2,localhost3, ..., localhost20 in hosts
+ - run: |
+ sudo bash -c 'for i in {2..20}; do echo 127.0.0.${i} localhost${i} >> /etc/hosts; done'
+ cat /etc/hosts
jobs:
cassandra-analytics-core-spark2-2_11-jdk8:
docker:
- image: cimg/openjdk:8.0
steps:
+ - install_common
- checkout
- run:
name: Build and test against Spark 2 Scala 2.11 JDK8
command: |
- git clone --single-branch --branch trunk https://github.com/apache/cassandra-sidecar.git
- ./cassandra-sidecar/gradlew --project-dir=./cassandra-sidecar -Pversion=1.0.0-jdk8-local :vertx-client-shaded:publishToMavenLocal
+ ./scripts/build-dependencies.sh
./gradlew --stacktrace clean assemble check
environment:
SPARK_VERSION: "2"
@@ -35,19 +60,26 @@ jobs:
JDK_VERSION: "1.8"
- store_artifacts:
- path: build/reports
+ path: build/test-reports
destination: test-reports
+ - store_artifacts:
+ path: build/reports
+ destination: reports
+
+ - store_test_results:
+ path: build/test-reports
+
cassandra-analytics-core-spark2-2_12-jdk8:
docker:
- image: cimg/openjdk:8.0
steps:
+ - install_common
- checkout
- run:
name: Build and test against Spark 2 Scala 2.12 JDK8
command: |
- git clone --single-branch --branch trunk https://github.com/apache/cassandra-sidecar.git
- ./cassandra-sidecar/gradlew --project-dir=./cassandra-sidecar -Pversion=1.0.0-jdk8-local :vertx-client-shaded:publishToMavenLocal
+ ./scripts/build-dependencies.sh
./gradlew --stacktrace clean assemble check
environment:
SPARK_VERSION: "2"
@@ -55,19 +87,26 @@ jobs:
JDK_VERSION: "1.8"
- store_artifacts:
- path: build/reports
+ path: build/test-reports
destination: test-reports
+ - store_artifacts:
+ path: build/reports
+ destination: reports
+
+ - store_test_results:
+ path: build/test-reports
+
cassandra-analytics-core-spark3-2_12-jdk11:
docker:
- image: cimg/openjdk:11.0
steps:
+ - install_common
- checkout
- run:
name: Build and test against Spark 3 Scala 2.12 JDK11
command: |
- git clone --single-branch --branch trunk https://github.com/apache/cassandra-sidecar.git
- ./cassandra-sidecar/gradlew --project-dir=./cassandra-sidecar -Pversion=1.0.0-local :vertx-client-shaded:publishToMavenLocal
+ CASSANDRA_USE_JDK11=true ./scripts/build-dependencies.sh
./gradlew --stacktrace clean assemble check
environment:
SPARK_VERSION: "3"
@@ -75,19 +114,26 @@ jobs:
JDK_VERSION: "11"
- store_artifacts:
- path: build/reports
+ path: build/test-reports
destination: test-reports
+ - store_artifacts:
+ path: build/reports
+ destination: reports
+
+ - store_test_results:
+ path: build/test-reports
+
cassandra-analytics-core-spark3-2_13-jdk11:
docker:
- image: cimg/openjdk:11.0
steps:
+ - install_common
- checkout
- run:
name: Build and test against Spark 3 Scala 2.13 JDK11
command: |
- git clone --single-branch --branch trunk https://github.com/apache/cassandra-sidecar.git
- ./cassandra-sidecar/gradlew --project-dir=./cassandra-sidecar -Pversion=1.0.0-local :vertx-client-shaded:publishToMavenLocal
+ CASSANDRA_USE_JDK11=true ./scripts/build-dependencies.sh
./gradlew --stacktrace clean assemble check
environment:
SPARK_VERSION: "3"
@@ -95,9 +141,16 @@ jobs:
JDK_VERSION: "11"
- store_artifacts:
- path: build/reports
+ path: build/test-reports
destination: test-reports
+ - store_artifacts:
+ path: build/reports
+ destination: reports
+
+ - store_test_results:
+ path: build/test-reports
+
workflows:
version: 2
build-and-test:
diff --git a/.gitignore b/.gitignore
index 80c13ed..a57705d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -100,3 +100,10 @@ cassandra-analytics-build.properties
# MacOS
.DS_Store
cassandra-analytics-core/dtest-jars
+cassandra-analytics-integration-tests/dtest-jars
+
+# Built by the build - ignore
+.rat-excludes.txt
+dependencies/
+scripts/target/
+scripts/dependency-reduced-pom.xml
\ No newline at end of file
diff --git a/CHANGES.txt b/CHANGES.txt
index a26e455..31a4ed2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
1.0.0
+ * Use in-jvm dtest framework from Sidecar for testing (CASSANDRA-18759)
* Cassandra Analytics Start-Up Validation (CASSANDRA-18810)
* Expose per partition on-disk usage through new DataFrame that utilizes the Index.db SSTable file components (CASSANDRA-18683)
* Fix bulk writes with Buffered RowBufferMode (CASSANDRA-18692)
diff --git a/DEV-README.md b/DEV-README.md
index 89791d1..2a2f818 100644
--- a/DEV-README.md
+++ b/DEV-README.md
@@ -24,14 +24,42 @@ Cassandra Analytics supports Spark 2 (Scala 2.11 and 2.12) and Spark 3 (Scala 2.
This project uses Gradle as the dependency management and build framework.
+## Dependencies
+This library depends on both the [Cassandra Sidecar](https://github.com/apache/cassandra-sidecar) (test and production)
+and shaded in-jvm dtest jars from [Cassandra](https://github.com/apache/cassandra) (testing only).
+Because these artifacts are not published by the Cassandra project, we have provided a script to build them locally.
+
+NOTE: If you are working on multiple projects that depend on the Cassandra Sidecar and in-jvm dtest dependencies,
+you can share those artifacts by setting the `CASSANDRA_DEP_DIR` environment variable to a shared directory
+and dependencies will build there instead of local to the project.
+
+In order to build the necessary dependencies, please run the following:
+
+```shell
+./scripts/build-dependencies.sh
+```
+
+This will build both the necessary dtest jars and the sidecar libraries/package necessary for build and test.
+You can also skip either the dtest jar build or the sidecar build by setting the following
+environment variables to `true`:
+
+```shell
+SKIP_DTEST_JAR_BUILD=true SKIP_SIDECAR_BUILD=true ./scripts/build-dependencies.sh
+```
+
+Note that `build-dependencies.sh` attempts to pull the latest from branches specified in the `BRANCHES` environment
+variable for Cassandra dtest jars, and trunk for the sidecar.
+
## Building
+Once you've built the dependencies, you're ready to build the analytics project.
+
Cassandra Analytics will build for Spark 2 and Scala 2.11 by default.
Navigate to the top-level directory for this project:
```shell
-./gradlew clean package
+./gradlew clean assemble
```
### Spark 2 and Scala 2.12
@@ -40,7 +68,7 @@ To build for Scala 2.12, set the profile by exporting `SCALA_VERSION=2.12`:
```shell
export SCALA_VERSION=2.12
-./gradlew clean package
+./gradlew clean assemble
```
### Spark 3 and Scala 2.12
@@ -50,7 +78,7 @@ To build for Spark 3 and Scala 2.12, export both `SCALA_VERSION=2.12` and `SPARK
```shell
export SCALA_VERSION=2.12
export SPARK_VERSION=3
-./gradlew clean package
+./gradlew clean assemble
```
### Git hooks (optional)
diff --git a/README.md b/README.md
index 0ce2eb8..536bbb0 100644
--- a/README.md
+++ b/README.md
@@ -47,9 +47,7 @@ val df = sparkSession.read.format("org.apache.cassandra.spark.sparksql.Cassandra
The Cassandra Spark Bulk Writer allows for high-speed data ingest to Cassandra clusters running Cassandra 3.0 and 4.0.
-If you are a consumer of the Cassandra Spark Bulk Writer, please see our end-user documentation: usage instructions, FAQs, troubleshooting guides, and release notes.
-
-Developers interested in contributing to the SBW, please see the [DEV-README](DEV-README.md).
+Developers interested in contributing to the Analytics library, please see the [DEV-README](DEV-README.md).
## Getting Started
diff --git a/build.gradle b/build.gradle
index eb689a4..a9fad25 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1,3 +1,5 @@
+import org.nosphere.apache.rat.RatTask
+
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -23,10 +25,10 @@ import java.nio.file.Paths
plugins {
id 'java'
id 'java-library'
- id 'com.github.johnrengelman.shadow' version '7.1.2'
+ id 'com.github.johnrengelman.shadow' version '8.1.1'
// Release Audit Tool (RAT) plugin for checking project licenses
- id("org.nosphere.apache.rat") version "0.8.0"
+ id("org.nosphere.apache.rat") version "0.8.1"
}
repositories {
@@ -42,6 +44,13 @@ println("Spark version: ${ext.sparkLabel}")
ext.jdkLabel = System.getenv("JDK_VERSION") ?: "${analyticsJDKLevel}"
println("Java source/target compatibility level: ${ext.jdkLabel}")
+ext.dependencyLocation = (System.getenv("CASSANDRA_DEP_DIR") ?: "${rootDir}/dependencies") + "/"
+
+
+def buildDir = layout.buildDirectory.get().asFile.toPath()
+def ratExcludeFilePath = buildDir.resolve(".rat-excludes.txt")
+System.out.println("Rat exclude file:" + ratExcludeFilePath)
+
def profile = "profiles/scala-${ext.scalaLabel}-spark-${ext.sparkLabel}-jdk-${ext.jdkLabel}.gradle"
if (!file(profile).exists()) {
throw new InvalidUserDataException("Profile ${profile} does not exist, which indicates this combination of Scala, Spark, and Java is unsupported.\n" +
@@ -53,6 +62,11 @@ apply(from: profile)
apply(plugin: 'idea')
+
+// Force checkstyle and rat to run before test tasks for faster feedback
+def codeCheckTasks = task("codeCheckTasks")
+
+
tasks.register('copyCodeStyle', Copy) {
from('ide/idea/codeStyleSettings.xml')
into('.idea')
@@ -68,10 +82,14 @@ tasks.idea.dependsOn(tasks.copyInspections)
tasks.register('buildIgnoreRatList', Exec) {
description 'Builds a list of ignored files for the rat task from the unversioned git files'
- commandLine 'bash', '-c', 'git clean --force -d --dry-run -x | cut -c 14-'
+ commandLine 'bash', '-c', 'git clean --force -d --dry-run -x | grep -v "Would skip" | cut -c 14- '
+// +
+// '&& ' +
+// 'git clean --force -d --dry-run -x | grep "Would skip" | cut -c 23-\n'
doFirst {
- Files.createDirectories(file("${buildDir}").toPath())
- standardOutput new FileOutputStream("${buildDir}/.rat-excludes.txt")
+ // make sure the build directory exists so rat task can run independently
+ Files.createDirectories(buildDir)
+ standardOutput new FileOutputStream(ratExcludeFilePath.toFile())
}
// allows task to fail when git/cut commands are unavailable or fail
ignoreExitValue = true
@@ -80,7 +98,7 @@ tasks.register('buildIgnoreRatList', Exec) {
rat {
doFirst {
- def excludeFilePath = Paths.get("${buildDir}/.rat-excludes.txt")
+ def excludeFilePath = ratExcludeFilePath
def excludeLines = Files.readAllLines(excludeFilePath)
excludeLines.each { line ->
if (line.endsWith("/")) {
@@ -109,6 +127,13 @@ rat {
// resource files for test
excludes.add("**/test**/resources/**")
+ // resources
+ excludes.add("**/resources/sidecar.version")
+ excludes.add("**/dependencies/**")
+
+ // Rat excludes file, one directive per line
+ excludeFile.set(ratExcludeFilePath.toFile())
+
// XML, TXT and HTML reports directory, defaults to 'build/reports/rat'
reportDir.set(file("build/reports/rat"))
}
@@ -124,6 +149,20 @@ subprojects {
sourceCompatibility = "${project.rootProject.ext.jdkLabel}"
targetCompatibility = "${project.rootProject.ext.jdkLabel}"
+ configurations {
+ configureEach {
+ resolutionStrategy {
+ // Force set the jackson version that matches with the one from spark's dependencies.
+ // Do not favor newer jackson version from others, because that jackson module scala has a strict check
+ // on the paired jackson dependency.
+ force("com.fasterxml.jackson.module:jackson-module-scala_${scalaMajorVersion}:${jacksonScalaModuleVersion}")
+ force("com.fasterxml.jackson.core:jackson-core:${jacksonVersion}")
+ force("com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}")
+ force("com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}")
+ }
+ }
+ }
+
tasks.withType(JavaCompile) {
if ("${project.rootProject.ext.jdkLabel}" == '1.8') {
// Unfortunately, we can't use release here. We can only use public APIs when using release.
@@ -151,8 +190,15 @@ subprojects {
repositories {
mavenCentral()
+ mavenLocal {
+ url Paths.get(dependencyLocation)
+ content {
+ includeModule("org.apache.cassandra", "cassandra-sidecar")
+ includeGroup("org.apache.cassandra.sidecar")
+ }
+ }
// uncomment the line below for local development
- mavenLocal()
+ // mavenLocal()
}
dependencies {
@@ -197,4 +243,13 @@ subprojects {
println("JVM arguments for $project.name are $allJvmArgs")
}
}
+
+ codeCheckTasks.dependsOn(tasks.withType(Checkstyle))
+ codeCheckTasks.dependsOn(tasks.withType(RatTask))
+
+ tasks.withType(Test) {
+ shouldRunAfter(codeCheckTasks)
+ shouldRunAfter(tasks.withType(Checkstyle))
+ shouldRunAfter(tasks.withType(RatTask))
+ }
}
diff --git a/cassandra-analytics-core-example/README.md b/cassandra-analytics-core-example/README.md
index 89122c9..0b86c6f 100644
--- a/cassandra-analytics-core-example/README.md
+++ b/cassandra-analytics-core-example/README.md
@@ -68,9 +68,11 @@ Verify that your Cassandra cluster is configured and running successfully.
In this step, we will clone and configure the Cassandra Sidecar project. Finally, we will run Sidecar which will be
connecting to our local Cassandra 3-node cluster.
+Note that when building the main project, you should have run the `./scripts/build-dependencies.sh` script,
+which would have cloned and built the sidecar into `./dependencies/sidecar-build`. Use that build to run the sidecar.
+
```shell
-git clone https://github.com/apache/cassandra-sidecar
-cd cassandra-sidecar
+cd ./dependencies/sidecar-build
```
Configure the `src/main/dist/sidecar.yaml` file for your local environment. You will most likely only need to configure
@@ -108,8 +110,8 @@ cassandra_instances:
I have a 3 node setup, so I configure Sidecar for those 3 nodes. CCM creates the Cassandra cluster under
`${HOME}/.ccm/test`, so I update my `data_dirs` and `staging_dir` configuration to use my local path.
-Finally, run Cassandra Sidecar, we skip running integration tests because we need docker for integration tests. You
-can opt to run integration tests if you have docker running in your local environment.
+Finally, run Cassandra Sidecar. We skip running integration tests because the integration tests take quite some time.
+You can, of course, choose to run them (and should when working on the sidecar the project itself).
```shell
user:~$ ./gradlew run -x integrationTest
@@ -216,16 +218,6 @@ CREATE TABLE spark_test.test
);
```
-The Cassandra Spark Bulk Analytics project depends on the Sidecar client. Since the Sidecar dependencies for the
-Analytics have not yet been published to maven central, we need to build them locally:
-
-```shell
-cd ${SIDECAR_REPOSITORY_HOME}
-./gradlew -Pversion=1.0.0-local :vertx-client-shaded:publishToMavenLocal
-```
-
-We use the suffix `-local` to denote that these artifacts have been built locally.
-
Finally, we are ready to run the example spark job:
```shell
diff --git a/cassandra-analytics-core-example/build.gradle b/cassandra-analytics-core-example/build.gradle
index 93bce75..b0bcfd3 100644
--- a/cassandra-analytics-core-example/build.gradle
+++ b/cassandra-analytics-core-example/build.gradle
@@ -29,8 +29,8 @@ repositories {
dependencies {
implementation(project(':cassandra-analytics-core'))
implementation(group: 'net.java.dev.jna', name: 'jna', version: '5.9.0')
- implementation(group: 'org.apache.spark', name: 'spark-core_2.12', version: '3.2.1')
- implementation(group: 'org.apache.spark', name: 'spark-sql_2.12', version: '3.2.1')
+ implementation(group: "${sparkGroupId}", name: "spark-core_${scalaMajorVersion}", version: "${project.rootProject.sparkVersion}")
+ implementation(group: "${sparkGroupId}", name: "spark-sql_${scalaMajorVersion}", version: "${project.rootProject.sparkVersion}")
}
application {
diff --git a/cassandra-analytics-core/build.gradle b/cassandra-analytics-core/build.gradle
index e1bd028..7bc4ed5 100644
--- a/cassandra-analytics-core/build.gradle
+++ b/cassandra-analytics-core/build.gradle
@@ -1,3 +1,5 @@
+import java.nio.file.Paths
+
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -42,11 +44,6 @@ project(':cassandra-analytics-core') {
fourzero {
description = 'Bridge to Cassandra 4.0 with all of its dependencies'
}
- all {
- resolutionStrategy {
- force("com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}")
- }
- }
}
dependencies {
@@ -59,11 +56,8 @@ project(':cassandra-analytics-core') {
runtimeOnly(group: 'net.java.dev.jna', name: 'jna', version: "${jnaVersion}")
runtimeOnly(group: 'net.java.dev.jna', name: 'jna-platform', version: "${jnaVersion}")
- // This dependency needs to be installed locally by building the Cassandra Sidecar project using the following command:
- // ./gradlew -Pversion=1.0.0-local :vertx-client-shaded:publishToMavenLocal
- implementation(group: "${sidecarClientGroup}", name: "${sidecarClientName}", version: "${sidecarVersion}") {
- exclude(group: 'com.fasterxml.jackson.core', module: 'jackson-annotations') // use provided annotations from spark
- }
+ // This dependency must be built by running `scripts/build-dependencies.sh`
+ implementation(group: "${sidecarClientGroup}", name: "${sidecarClientName}", version: "${sidecarVersion}")
if ("${scalaMajorVersion}" == "2.11") {
implementation(group: 'org.scala-lang.modules', name: "scala-java8-compat_2.11", version: '1.0.1', transitive: false)
@@ -134,19 +128,26 @@ project(':cassandra-analytics-core') {
dependsOn(tasks.jar)
classpath = project.sourceSets.test.output + configurations.testRuntimeClasspath + files(jar.archiveFile)
useJUnitPlatform()
+ reports {
+ def destDir = Paths.get(rootProject.rootDir.absolutePath, "build", "test-reports", "core").toFile()
+ junitXml {
+ enabled true
+ destination = destDir
+ }
+ html{
+ enabled true
+ destination = destDir
+ }
+ }
}
/* Start: JaCoCo check */
jacoco {
toolVersion = '0.8.4'
}
- task jacocoMerge(type: JacocoMerge) {
- destinationFile = file("$buildDir/jacoco/mergedFile.exec")
- executionData = files("$buildDir/jacoco/test.exec")
- }
+
jacocoTestReport {
- dependsOn(tasks.jacocoMerge)
- executionData(tasks.jacocoMerge.executionData)
+ executionData(files("$buildDir/jacoco/test.exec"))
reports {
html.required = true
html.destination(file("$buildDir/reports/jacocoHtml"))
@@ -155,8 +156,7 @@ project(':cassandra-analytics-core') {
}
}
jacocoTestCoverageVerification {
- dependsOn(tasks.jacocoMerge)
- executionData(tasks.jacocoMerge.executionData)
+ executionData(tasks.jacocoTestReport.executionData)
violationRules {
rule {
limit {
@@ -175,7 +175,7 @@ project(':cassandra-analytics-core') {
testArtifacts
}
task testJar(type: Jar) {
- baseName = "${project.name}-test"
+ archiveBaseName = "${project.name}-test"
from sourceSets.test.output
}
artifacts {
diff --git a/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/validation/KeyStoreValidation.java b/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/validation/KeyStoreValidation.java
index 4ee2faa..febb0c8 100644
--- a/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/validation/KeyStoreValidation.java
+++ b/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/validation/KeyStoreValidation.java
@@ -55,7 +55,7 @@ public class KeyStoreValidation implements StartupValidation
{
configured = configuration.hasKeystoreAndKeystorePassword();
type = configuration.getKeyStoreTypeOrDefault();
- password = configuration.getKeyStorePassword().toCharArray();
+ password = configuration.getKeyStorePassword() == null ? null : configuration.getKeyStorePassword().toCharArray();
stream = () -> configuration.getKeyStore();
}
@@ -69,6 +69,11 @@ public class KeyStoreValidation implements StartupValidation
throw new RuntimeException("KeyStore is not configured");
}
+ if (password == null)
+ {
+ throw new RuntimeException("Keystore password was not provided.");
+ }
+
KeyStore keyStore = KeyStore.getInstance(type);
keyStore.load(stream.get(), password);
if (keyStore.size() == 0)
diff --git a/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/validation/StartupValidator.java b/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/validation/StartupValidator.java
index f48fba0..9651389 100644
--- a/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/validation/StartupValidator.java
+++ b/cassandra-analytics-core/src/main/java/org/apache/cassandra/spark/validation/StartupValidator.java
@@ -19,8 +19,8 @@
package org.apache.cassandra.spark.validation;
-import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
@@ -33,10 +33,10 @@ import org.slf4j.LoggerFactory;
public final class StartupValidator
{
private static final Logger LOGGER = LoggerFactory.getLogger(StartupValidator.class);
- private static final StartupValidator INSTANCE = new StartupValidator();
+ private static final ThreadLocal<StartupValidator> INSTANCE = ThreadLocal.withInitial(StartupValidator::new);
private static final String DISABLE = "SKIP_STARTUP_VALIDATIONS";
- private final List<StartupValidation> validations = new ArrayList<>();
+ private final List<StartupValidation> validations = new CopyOnWriteArrayList<>();
private StartupValidator()
{
@@ -44,7 +44,7 @@ public final class StartupValidator
public static StartupValidator instance()
{
- return INSTANCE;
+ return INSTANCE.get();
}
public void register(StartupValidation validation)
@@ -60,15 +60,24 @@ public final class StartupValidator
public void perform()
{
- if (enabled())
+ try
{
- LOGGER.info("Performing startup validations");
- validations.forEach(StartupValidation::perform);
- LOGGER.info("Completed startup validations");
+ if (enabled())
+ {
+ LOGGER.info("Performing startup validations");
+ validations.forEach(StartupValidation::perform);
+ LOGGER.info("Completed startup validations");
+ }
+ else
+ {
+ LOGGER.info("Skipping startup validations");
+ }
}
- else
+ finally
{
- LOGGER.info("Skipping startup validations");
+ // Regardless of enabled status, we should still remove all of the validations
+ // because otherwise the list just keeps growing.
+ reset();
}
}
diff --git a/cassandra-analytics-core/src/test/java/org/apache/cassandra/spark/bulkwriter/MockBulkWriterContext.java b/cassandra-analytics-core/src/test/java/org/apache/cassandra/spark/bulkwriter/MockBulkWriterContext.java
index 33e0807..19ba2d6 100644
--- a/cassandra-analytics-core/src/test/java/org/apache/cassandra/spark/bulkwriter/MockBulkWriterContext.java
+++ b/cassandra-analytics-core/src/test/java/org/apache/cassandra/spark/bulkwriter/MockBulkWriterContext.java
@@ -22,12 +22,12 @@ package org.apache.cassandra.spark.bulkwriter;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.function.BiFunction;
import java.util.function.Predicate;
import java.util.function.Supplier;
@@ -77,11 +77,11 @@ public class MockBulkWriterContext implements BulkWriterContext, ClusterInfo, Jo
private Supplier<Long> timeProvider = System::currentTimeMillis;
private boolean skipClean = false;
public int refreshClusterInfoCallCount = 0; // CHECKSTYLE IGNORE: Public mutable field
- private final Map<CassandraInstance, List<UploadRequest>> uploads = new HashMap<>();
- private final Map<CassandraInstance, List<String>> commits = new HashMap<>();
+ private final Map<CassandraInstance, List<UploadRequest>> uploads = new ConcurrentHashMap<>();
+ private final Map<CassandraInstance, List<String>> commits = new ConcurrentHashMap<>();
final Pair<StructType, ImmutableMap<String, CqlField.CqlType>> validPair;
private final TableSchema schema;
- private final Set<CassandraInstance> cleanCalledForInstance = new HashSet<>();
+ private final Set<CassandraInstance> cleanCalledForInstance = Collections.synchronizedSet(new HashSet<>());
private boolean instancesAreAvailable = true;
private boolean cleanShouldThrow = false;
private final CassandraRing<RingInstance> ring;
@@ -292,7 +292,7 @@ public class MockBulkWriterContext implements BulkWriterContext, ClusterInfo, Jo
commits.compute(instance, (ignored, commitList) -> {
if (commitList == null)
{
- commitList = new ArrayList<>();
+ commitList = Collections.synchronizedList(new ArrayList<>());
}
commitList.add(migrationId);
return commitList;
diff --git a/cassandra-analytics-integration-framework/build.gradle b/cassandra-analytics-integration-framework/build.gradle
new file mode 100644
index 0000000..d1426b4
--- /dev/null
+++ b/cassandra-analytics-integration-framework/build.gradle
@@ -0,0 +1,75 @@
+import java.nio.file.Paths
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
+
+project(':cassandra-analytics-integration-framework') {
+ apply(plugin: 'java-library')
+ apply(plugin: 'com.github.johnrengelman.shadow')
+
+ ext.dtestJar = System.getenv("DTEST_JAR") ?: "dtest-4.1.4.jar" // latest supported Cassandra build is 4.1
+ def dtestJarFullPath = "${dependencyLocation}${ext.dtestJar}"
+
+ dependencies {
+
+ // These dependencies (sidecar + dtest jars) must be built by running `scripts/build-dependencies.sh`
+ implementation(group: 'org.apache.cassandra.sidecar', name: 'common', version: "${sidecarVersion}")
+ implementation(group: 'org.apache.cassandra.sidecar', name: 'adapters-base', version: "${sidecarVersion}")
+ // The server itself
+ implementation (group: "org.apache.cassandra", name: "cassandra-sidecar", version: "${sidecarVersion}") {
+ exclude(group: 'org.apache.logging.log4j')
+ exclude(group: 'org.slf4j')
+ exclude(group: 'ch.qos.logback')
+ }
+
+ File dtestJarFile = new File(dtestJarFullPath)
+ assert dtestJarFile.exists()
+ println("Using DTest jar: " + dtestJarFullPath)
+ shadow(files("${dtestJarFullPath}"))
+ shadow("org.apache.cassandra:dtest-api:0.0.16")
+ // Needed by the Cassandra dtest framework
+ // JUnit
+ api("org.junit.jupiter:junit-jupiter-api:${project.junitVersion}")
+ api("org.junit.jupiter:junit-jupiter-params:${project.junitVersion}")
+ api("org.junit.jupiter:junit-jupiter-engine:${project.junitVersion}")
+
+ api("org.junit.vintage:junit-vintage-engine:${junitVersion}")
+ api('org.mockito:mockito-inline:4.10.0')
+ api("org.assertj:assertj-core:3.24.2")
+
+ shadow('com.datastax.cassandra:cassandra-driver-core:3.9.0')
+
+ shadow("com.google.guava:guava:27.0.1-jre")
+ shadow('com.google.inject:guice:4.2.2')
+ shadow("io.vertx:vertx-web:${project.vertxVersion}") {
+ exclude group: 'junit', module: 'junit'
+ }
+ shadow("io.vertx:vertx-web-client:${project.vertxVersion}")
+ shadow("io.vertx:vertx-junit5:${project.vertxVersion}")
+ }
+
+ shadowJar {
+ zip64 true
+ relocate('org.apache.cassandra.sidecar', 'o.a.c.analytics.sidecar.shaded.testing') {
+ exclude '%regex[org.apache.cassandra.sidecar.testing.*]'
+ }
+ }
+}
diff --git a/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/sidecar/testing/CassandraSidecarTestContext.java b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/sidecar/testing/CassandraSidecarTestContext.java
new file mode 100644
index 0000000..d21b5c7
--- /dev/null
+++ b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/sidecar/testing/CassandraSidecarTestContext.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sidecar.testing;
+
+import java.net.InetSocketAddress;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import com.datastax.driver.core.NettyOptions;
+import com.datastax.driver.core.Session;
+import org.apache.cassandra.distributed.UpgradeableCluster;
+import org.apache.cassandra.distributed.api.IInstanceConfig;
+import org.apache.cassandra.distributed.api.IUpgradeableInstance;
+import org.apache.cassandra.distributed.shared.JMXUtil;
+import org.apache.cassandra.sidecar.adapters.base.CassandraFactory;
+import org.apache.cassandra.sidecar.cluster.InstancesConfig;
+import org.apache.cassandra.sidecar.cluster.InstancesConfigImpl;
+import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata;
+import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadataImpl;
+import org.apache.cassandra.sidecar.common.CQLSessionProvider;
+import org.apache.cassandra.sidecar.common.CassandraAdapterDelegate;
+import org.apache.cassandra.sidecar.common.CassandraVersionProvider;
+import org.apache.cassandra.sidecar.common.JmxClient;
+import org.apache.cassandra.sidecar.common.SimpleCassandraVersion;
+import org.apache.cassandra.sidecar.common.dns.DnsResolver;
+import org.apache.cassandra.sidecar.common.utils.SidecarVersionProvider;
+import org.apache.cassandra.testing.AbstractCassandraTestContext;
+import org.assertj.core.api.Assertions;
+
+/**
+ * Passed to integration tests.
+ */
+public class CassandraSidecarTestContext implements AutoCloseable
+{
+ public final SimpleCassandraVersion version;
+ private final CassandraVersionProvider versionProvider;
+ private final DnsResolver dnsResolver;
+ private final AbstractCassandraTestContext abstractCassandraTestContext;
+ public InstancesConfig instancesConfig;
+ private List<CQLSessionProvider> sessionProviders;
+ private List<JmxClient> jmxClients;
+ private static final SidecarVersionProvider svp = new SidecarVersionProvider("/sidecar.version");
+ private final List<InstanceConfigListener> instanceConfigListeners;
+
+ private CassandraSidecarTestContext(AbstractCassandraTestContext abstractCassandraTestContext,
+ SimpleCassandraVersion version,
+ CassandraVersionProvider versionProvider,
+ DnsResolver dnsResolver)
+ {
+ this.instanceConfigListeners = new ArrayList<>();
+ this.abstractCassandraTestContext = abstractCassandraTestContext;
+ this.version = version;
+ this.versionProvider = versionProvider;
+ this.dnsResolver = dnsResolver;
+ }
+
+ public static CassandraSidecarTestContext from(AbstractCassandraTestContext cassandraTestContext,
+ DnsResolver dnsResolver)
+ {
+ org.apache.cassandra.testing.SimpleCassandraVersion rootVersion = cassandraTestContext.version;
+ SimpleCassandraVersion versionParsed = SimpleCassandraVersion.create(rootVersion.major,
+ rootVersion.minor,
+ rootVersion.patch);
+ CassandraVersionProvider versionProvider = cassandraVersionProvider(dnsResolver);
+ return new CassandraSidecarTestContext(cassandraTestContext,
+ versionParsed,
+ versionProvider,
+ dnsResolver);
+ }
+
+ public static CassandraVersionProvider cassandraVersionProvider(DnsResolver dnsResolver)
+ {
+ return new CassandraVersionProvider.Builder()
+ .add(new CassandraFactory(dnsResolver, svp.sidecarVersion())).build();
+ }
+
+ public void registerInstanceConfigListener(InstanceConfigListener listener)
+ {
+ this.instanceConfigListeners.add(listener);
+ }
+
+ public AbstractCassandraTestContext cassandraTestContext()
+ {
+ return abstractCassandraTestContext;
+ }
+
+ public boolean isClusterBuilt()
+ {
+ return abstractCassandraTestContext.cluster() != null;
+ }
+
+ public UpgradeableCluster cluster()
+ {
+ if (abstractCassandraTestContext.cluster() == null)
+ {
+ throw new RuntimeException("The cluster must be built before it can be used");
+ }
+ return abstractCassandraTestContext.cluster();
+ }
+
+ public InstancesConfig instancesConfig()
+ {
+ if (instancesConfig == null
+ || instancesConfig.instances().size() != cluster().size()) // rebuild instances config if cluster changed
+ {
+ setInstancesConfig();
+ }
+ return this.instancesConfig;
+ }
+
+ public Session session()
+ {
+ return session(0);
+ }
+
+ public Session session(int instance)
+ {
+ if (sessionProviders == null)
+ {
+ setInstancesConfig();
+ }
+ return this.sessionProviders.get(instance).localCql();
+ }
+
+ @Override
+ public String toString()
+ {
+ return "CassandraTestContext{" +
+ ", version=" + version +
+ ", cluster=" + abstractCassandraTestContext.cluster() +
+ '}';
+ }
+
+ @Override
+ public void close()
+ {
+ if (sessionProviders != null)
+ {
+ sessionProviders.forEach(CQLSessionProvider::close);
+ }
+ if (instancesConfig != null)
+ {
+ instancesConfig.instances().forEach(instance -> instance.delegate().close());
+ }
+ }
+
+ public JmxClient jmxClient()
+ {
+ return jmxClient(0);
+ }
+
+ private JmxClient jmxClient(int instance)
+ {
+ if (jmxClients == null)
+ {
+ setInstancesConfig();
+ }
+ return jmxClients.get(instance);
+ }
+
+ /**
+ * A listener for {@link InstancesConfig} state changes
+ */
+ public interface InstanceConfigListener
+ {
+ void onInstancesConfigChange(InstancesConfig instancesConfig);
+ }
+
+ private void setInstancesConfig()
+ {
+ this.instancesConfig = buildInstancesConfig(versionProvider, dnsResolver);
+ for (InstanceConfigListener listener : instanceConfigListeners)
+ {
+ listener.onInstancesConfigChange(this.instancesConfig);
+ }
+ }
+
+ private InstancesConfig buildInstancesConfig(CassandraVersionProvider versionProvider,
+ DnsResolver dnsResolver)
+ {
+ UpgradeableCluster cluster = cluster();
+ List<InstanceMetadata> metadata = new ArrayList<>();
+ sessionProviders = new ArrayList<>();
+ jmxClients = new ArrayList<>();
+ for (int i = 0; i < cluster.size(); i++)
+ {
+ IUpgradeableInstance instance = cluster.get(i + 1); // 1-based indexing to match node names;
+ IInstanceConfig config = instance.config();
+ String hostName = JMXUtil.getJmxHost(config);
+ int nativeTransportPort = tryGetIntConfig(config, "native_transport_port", 9042);
+ InetSocketAddress address = InetSocketAddress.createUnresolved(hostName,
+ nativeTransportPort);
+ CQLSessionProvider sessionProvider = new CQLSessionProvider(address, new NettyOptions());
+ this.sessionProviders.add(sessionProvider);
+ JmxClient jmxClient = new JmxClient(hostName, config.jmxPort());
+ this.jmxClients.add(jmxClient);
+
+ CassandraAdapterDelegate delegate = new CassandraAdapterDelegate(versionProvider,
+ sessionProvider,
+ jmxClient,
+ "1.0-TEST");
+
+ String[] dataDirectories = (String[]) config.get("data_file_directories");
+ // Use the parent of the first data directory as the staging directory
+ Path dataDirParentPath = Paths.get(dataDirectories[0]).getParent();
+ // If the cluster has not started yet, the node's root directory doesn't exist yet
+ Assertions.assertThat(dataDirParentPath).isNotNull();
+ Path stagingPath = dataDirParentPath.resolve("staging");
+ String uploadsStagingDirectory = stagingPath.toFile().getAbsolutePath();
+ metadata.add(InstanceMetadataImpl.builder()
+ .id(i + 1)
+ .host(config.broadcastAddress().getAddress().getHostAddress())
+ .port(nativeTransportPort)
+ .dataDirs(Arrays.asList(dataDirectories))
+ .stagingDir(uploadsStagingDirectory)
+ .delegate(delegate)
+ .build());
+ }
+ return new InstancesConfigImpl(metadata, dnsResolver);
+ }
+
+ private static int tryGetIntConfig(IInstanceConfig config, String configName, int defaultValue)
+ {
+ try
+ {
+ return config.getInt(configName);
+ }
+ catch (NullPointerException npe)
+ {
+ return defaultValue;
+ }
+ }
+}
diff --git a/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/sidecar/testing/IntegrationTestBase.java b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/sidecar/testing/IntegrationTestBase.java
new file mode 100644
index 0000000..52d8937
--- /dev/null
+++ b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/sidecar/testing/IntegrationTestBase.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sidecar.testing;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.datastax.driver.core.KeyspaceMetadata;
+import com.datastax.driver.core.Session;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.util.Modules;
+import io.vertx.core.Vertx;
+import io.vertx.core.http.HttpServer;
+import io.vertx.ext.web.client.WebClient;
+import io.vertx.junit5.VertxTestContext;
+import org.apache.cassandra.sidecar.MainModule;
+import org.apache.cassandra.sidecar.cluster.InstancesConfig;
+import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata;
+import org.apache.cassandra.sidecar.common.data.QualifiedTableName;
+import org.apache.cassandra.sidecar.common.dns.DnsResolver;
+import org.apache.cassandra.testing.AbstractCassandraTestContext;
+import org.assertj.core.api.Assertions;
+
+/**
+ * Base class for integration test.
+ * Start an in-jvm dtest cluster at the beginning of each test, and
+ * teardown the cluster after each test.
+ */
+public abstract class IntegrationTestBase
+{
+ private static final int MAX_KEYSPACE_TABLE_WAIT_ATTEMPTS = 100;
+ private static final long MAX_KEYSPACE_TABLE_TIME = 100L;
+ protected Logger logger = LoggerFactory.getLogger(this.getClass());
+ protected Vertx vertx;
+ protected HttpServer server;
+
+ protected static final String TEST_KEYSPACE = "testkeyspace";
+ private static final String TEST_TABLE_PREFIX = "testtable";
+
+ protected static final int DEFAULT_RF = 3;
+ private static final AtomicInteger TEST_TABLE_ID = new AtomicInteger(0);
+ protected CassandraSidecarTestContext sidecarTestContext;
+
+ @BeforeEach
+ void setup(AbstractCassandraTestContext cassandraTestContext) throws InterruptedException
+ {
+ sidecarTestContext = CassandraSidecarTestContext.from(cassandraTestContext, DnsResolver.DEFAULT);
+ Injector injector = Guice.createInjector(Modules
+ .override(new MainModule())
+ .with(new IntegrationTestModule(this.sidecarTestContext)));
+ server = injector.getInstance(HttpServer.class);
+ vertx = injector.getInstance(Vertx.class);
+
+ VertxTestContext context = new VertxTestContext();
+ server.listen(0, context.succeeding(p -> {
+ if (sidecarTestContext.isClusterBuilt())
+ {
+ healthCheck(sidecarTestContext.instancesConfig());
+ }
+ sidecarTestContext.registerInstanceConfigListener(IntegrationTestBase::healthCheck);
+ context.completeNow();
+ }));
+
+ context.awaitCompletion(5, TimeUnit.SECONDS);
+ }
+
+ @AfterEach
+ void tearDown() throws InterruptedException
+ {
+ final CountDownLatch closeLatch = new CountDownLatch(1);
+ server.close(res -> closeLatch.countDown());
+ vertx.close();
+ if (closeLatch.await(60, TimeUnit.SECONDS))
+ {
+ logger.info("Close event received before timeout.");
+ }
+ else
+ {
+ logger.error("Close event timed out.");
+ }
+ sidecarTestContext.close();
+ }
+
+ protected void testWithClient(VertxTestContext context, Consumer<WebClient> tester) throws Exception
+ {
+ WebClient client = WebClient.create(vertx);
+
+ tester.accept(client);
+
+ // wait until the test completes
+ Assertions.assertThat(context.awaitCompletion(30, TimeUnit.SECONDS)).isTrue();
+ }
+
+ protected void createTestKeyspace()
+ {
+ createTestKeyspace(ImmutableMap.of("datacenter1", 1));
+ }
+
+ protected void createTestKeyspace(Map<String, Integer> rf)
+ {
+ Session session = maybeGetSession();
+ session.execute("CREATE KEYSPACE " + TEST_KEYSPACE +
+ " WITH REPLICATION = { 'class' : 'NetworkTopologyStrategy', " + generateRfString(rf) + " };");
+ }
+
+ private String generateRfString(Map<String, Integer> dcToRf)
+ {
+ return dcToRf.entrySet().stream().map(e -> String.format("'%s':%d", e.getKey(), e.getValue()))
+ .collect(Collectors.joining(","));
+ }
+
+ protected QualifiedTableName createTestTable(String createTableStatement)
+ {
+ Session session = maybeGetSession();
+ QualifiedTableName tableName = uniqueTestTableFullName();
+ session.execute(String.format(createTableStatement, tableName));
+ return tableName;
+ }
+
+ protected Session maybeGetSession()
+ {
+ Session session = sidecarTestContext.session();
+ Assertions.assertThat(session).isNotNull();
+ return session;
+ }
+
+ private static QualifiedTableName uniqueTestTableFullName()
+ {
+ return new QualifiedTableName(TEST_KEYSPACE, TEST_TABLE_PREFIX + TEST_TABLE_ID.getAndIncrement());
+ }
+
+ public List<Path> findChildFile(CassandraSidecarTestContext context, String hostname, String target)
+ {
+ InstanceMetadata instanceConfig = context.instancesConfig().instanceFromHost(hostname);
+ List<String> parentDirectories = instanceConfig.dataDirs();
+
+ return parentDirectories.stream().flatMap(s -> findChildFile(Paths.get(s), target).stream())
+ .collect(Collectors.toList());
+ }
+
+ private List<Path> findChildFile(Path path, String target)
+ {
+ try (Stream<Path> walkStream = Files.walk(path))
+ {
+ return walkStream.filter(p -> p.toString().endsWith(target)
+ || p.toString().contains("/" + target + "/"))
+ .collect(Collectors.toList());
+ }
+ catch (IOException e)
+ {
+ return Collections.emptyList();
+ }
+ }
+
+ private static void healthCheck(InstancesConfig instancesConfig)
+ {
+ instancesConfig.instances()
+ .forEach(instanceMetadata -> instanceMetadata.delegate().healthCheck());
+ }
+
+ /**
+ * Waits for the specified keyspace/table to be available.
+ * Emperically, this loop usually executes either zero or one time before completing.
+ * However, we set a fairly high number of retries to account for variability in build machines.
+ *
+ * @param keyspaceName the keyspace for which to wait
+ * @param tableName the table in the keyspace for which to wait
+ */
+ protected void waitForKeyspaceAndTable(String keyspaceName, String tableName)
+ {
+ int numInstances = sidecarTestContext.instancesConfig().instances().size();
+ int retries = MAX_KEYSPACE_TABLE_WAIT_ATTEMPTS;
+ boolean sidecarMissingSchema = true;
+ while (sidecarMissingSchema && retries-- > 0)
+ {
+ sidecarMissingSchema = false;
+ for (int i = 0; i < numInstances; i++)
+ {
+ KeyspaceMetadata keyspace = sidecarTestContext.session(i)
+ .getCluster()
+ .getMetadata()
+ .getKeyspace(keyspaceName);
+ sidecarMissingSchema |= (keyspace == null || keyspace.getTable(tableName) == null);
+ }
+ if (sidecarMissingSchema)
+ {
+ logger.info("Keyspace/table {}/{} not yet available - waiting...", keyspaceName, tableName);
+ Uninterruptibles.sleepUninterruptibly(MAX_KEYSPACE_TABLE_TIME, TimeUnit.MILLISECONDS);
+ }
+ else
+ {
+ return;
+ }
+ }
+ throw new RuntimeException(
+ String.format("Keyspace/table %s/%s did not become visible on all sidecar instances",
+ keyspaceName, tableName));
+ }
+}
diff --git a/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/sidecar/testing/IntegrationTestModule.java b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/sidecar/testing/IntegrationTestModule.java
new file mode 100644
index 0000000..9e65286
--- /dev/null
+++ b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/sidecar/testing/IntegrationTestModule.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sidecar.testing;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Provides;
+import com.google.inject.Singleton;
+import org.apache.cassandra.sidecar.cluster.InstancesConfig;
+import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata;
+import org.apache.cassandra.sidecar.config.SidecarConfiguration;
+import org.apache.cassandra.sidecar.config.yaml.ServiceConfigurationImpl;
+import org.apache.cassandra.sidecar.config.yaml.SidecarConfigurationImpl;
+
+/**
+ * Provides the basic dependencies for integration tests
+ */
+public class IntegrationTestModule extends AbstractModule
+{
+ private final CassandraSidecarTestContext cassandraTestContext;
+
+ public IntegrationTestModule(CassandraSidecarTestContext cassandraTestContext)
+ {
+ this.cassandraTestContext = cassandraTestContext;
+ }
+
+ @Provides
+ @Singleton
+ public InstancesConfig instancesConfig()
+ {
+ return new WrapperInstancesConfig(cassandraTestContext);
+ }
+
+ static class WrapperInstancesConfig implements InstancesConfig
+ {
+ private final CassandraSidecarTestContext cassandraTestContext;
+
+ WrapperInstancesConfig(CassandraSidecarTestContext cassandraTestContext)
+ {
+ this.cassandraTestContext = cassandraTestContext;
+ }
+
+ /**
+ * @return metadata of instances owned by the sidecar
+ */
+ public List<InstanceMetadata> instances()
+ {
+ if (cassandraTestContext.isClusterBuilt())
+ {
+ return cassandraTestContext.instancesConfig.instances();
+ }
+ return Collections.emptyList();
+ }
+
+ /**
+ * Lookup instance metadata by id.
+ *
+ * @param id instance's id
+ * @return instance meta information
+ * @throws NoSuchElementException when the instance with {@code id} does not exist
+ */
+ public InstanceMetadata instanceFromId(int id) throws NoSuchElementException
+ {
+ return cassandraTestContext.instancesConfig.instanceFromId(id);
+ }
+
+ /**
+ * Lookup instance metadata by host name.
+ *
+ * @param host host address of instance
+ * @return instance meta information
+ * @throws NoSuchElementException when the instance for {@code host} does not exist
+ */
+ public InstanceMetadata instanceFromHost(String host) throws NoSuchElementException
+ {
+ return cassandraTestContext.instancesConfig.instanceFromHost(host);
+ }
+ }
+
+ @Provides
+ @Singleton
+ public SidecarConfiguration sidecarConfiguration()
+ {
+ return new SidecarConfigurationImpl(new ServiceConfigurationImpl("127.0.0.1"));
+ }
+}
diff --git a/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/AbstractCassandraTestContext.java b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/AbstractCassandraTestContext.java
new file mode 100644
index 0000000..cde163e
--- /dev/null
+++ b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/AbstractCassandraTestContext.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.testing;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.distributed.UpgradeableCluster;
+
+/**
+ * The base class for all CassandraTestContext implementations
+ */
+public abstract class AbstractCassandraTestContext implements AutoCloseable
+{
+ private static final Logger LOGGER = LoggerFactory.getLogger(AbstractCassandraTestContext.class);
+
+ public final SimpleCassandraVersion version;
+ protected UpgradeableCluster cluster;
+
+ public CassandraIntegrationTest annotation;
+
+ public AbstractCassandraTestContext(SimpleCassandraVersion version,
+ UpgradeableCluster cluster,
+ CassandraIntegrationTest annotation)
+ {
+ this.version = version;
+ this.cluster = cluster;
+ this.annotation = annotation;
+ }
+
+ public AbstractCassandraTestContext(SimpleCassandraVersion version,
+ CassandraIntegrationTest annotation)
+ {
+ this.version = version;
+ this.annotation = annotation;
+ }
+
+ public UpgradeableCluster cluster()
+ {
+ return cluster;
+ }
+
+ @Override
+ public void close()
+ {
+ if (cluster != null)
+ {
+ try
+ {
+ cluster.close();
+ }
+ catch (Exception ex)
+ {
+ // Because different classloaders or different jars may be used to load this class, we can't
+ // actually catch the specific ShutdownException.
+ if (ex.getClass().getCanonicalName() == "org.apache.cassandra.distributed.shared.ShutdownException")
+ {
+ LOGGER.warn("Encountered shutdown exception which closing the cluster", ex);
+ }
+ else
+ {
+ throw ex;
+ }
+ }
+ }
+ }
+}
diff --git a/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/CassandraIntegrationTest.java b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/CassandraIntegrationTest.java
new file mode 100644
index 0000000..efa7a12
--- /dev/null
+++ b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/CassandraIntegrationTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.testing;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.util.function.Consumer;
+
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+/**
+ * Interface to mark an integration test which should be run against multiple Cassandra versions
+ */
+@TestTemplate
+@Target({ ElementType.ANNOTATION_TYPE, ElementType.METHOD })
+@Retention(RetentionPolicy.RUNTIME)
+@Tag("integrationTest")
+@ExtendWith(CassandraTestTemplate.class)
+public @interface CassandraIntegrationTest
+{
+ /**
+ * Returns the number of initial nodes per datacenter for the integration tests. Defaults to 1 node per datacenter.
+ *
+ * @return the number of nodes per datacenter for the integration tests
+ */
+ int nodesPerDc() default 1;
+
+ /**
+ * Returns the number of nodes expected to be added by the end of the integration test. Defaults ot 0.
+ *
+ * @return the number of nodes the test expects to add for the integration test.
+ */
+ int newNodesPerDc() default 0;
+
+ /**
+ * Returns the number of datacenters to configure for the integration test. Defaults to 1 datacenter.
+ *
+ * @return the number of datacenters to configure for the integration test
+ */
+ int numDcs() default 1;
+
+ /**
+ * Returns the number of data directories to use per instance. Cassandra supports multiple data directories
+ * for each instance. Defaults to 1 data directory per instance.
+ *
+ * @return the number of data directories to use per instance
+ */
+ int numDataDirsPerInstance() default 1;
+
+ /**
+ * Returns whether gossip is enabled or disabled for the integration test. Defaults to {@code false}.
+ *
+ * @return whether gossip is enabled or disabled for the integration test
+ */
+ boolean gossip() default false;
+
+ /**
+ * Returns whether internode networking is enabled or disabled for the integration test. Defaults to {@code false}.
+ *
+ * @return whether internode networking is enabled or disabled for the integration test
+ */
+ boolean network() default false;
+
+ /**
+ * Returns whether JMX is enabled or disabled for the integration test. Defaults to {@code true}.
+ *
+ * @return whether JMX is enabled or disabled for the integration test
+ */
+ boolean jmx() default true;
+
+ /**
+ * Returns whether the native transport protocol is enabled or disabled for the integration test. Defaults to
+ * {@code true}.
+ *
+ * @return whether the native transport protocol is enabled or disabled for the integration test
+ */
+ boolean nativeTransport() default true;
+
+ /**
+ * Return whether the cluster should be started before the test begins.
+ * It may be necessary to delay start/start in a thread if using ByteBuddy-based
+ * interception of cluster startup.
+ * @return true, if the cluster should be started before the test starts, false otherwise
+ */
+ boolean startCluster() default true;
+
+ /**
+ * Return whether the cluster should be built, or to simply add the cluster builder to the context.
+ * This may be useful in cases where the test requires more complex cluster startup.
+ * If false, the test should take an instance of {@link ConfigurableCassandraTestContext}
+ * and call {@link ConfigurableCassandraTestContext#configureCluster(Consumer)}
+ * or {@link ConfigurableCassandraTestContext#configureAndStartCluster(Consumer)} to get the cluster.
+ * NOTE: This cluster object must be closed by the test as the framework doesn't have access to it.
+ * If true (the default), the test should take an instance of {@link CassandraTestContext}
+ * {@link CassandraTestContext#getCluster()} will contain the built cluster.
+ * @return true if the cluster should be built by the test framework, false otherwise
+ */
+ boolean buildCluster() default true;
+
+ /**
+ * If the integration test does not need to be run on each version of Cassandra, set this to false
+ * and it will be run only on the first version specified.
+ * @return true if the test should be run on all tested versions of Cassandra,
+ * false if it should be run on the first version.
+ */
+ boolean versionDependent() default true;
+}
diff --git a/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/CassandraTestContext.java b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/CassandraTestContext.java
new file mode 100644
index 0000000..8498776
--- /dev/null
+++ b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/CassandraTestContext.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.testing;
+
+import org.apache.cassandra.distributed.UpgradeableCluster;
+
+/**
+ * Passed to integration tests.
+ * See {@link CassandraIntegrationTest} for the required annotation
+ * See {@link CassandraTestTemplate} for the Test Template
+ */
+public class CassandraTestContext extends AbstractCassandraTestContext
+{
+
+ public CassandraTestContext(SimpleCassandraVersion version,
+ UpgradeableCluster cluster,
+ CassandraIntegrationTest annotation)
+ {
+ super(version, cluster, annotation);
+ }
+
+ @Override
+ public String toString()
+ {
+ return "CassandraTestContext{"
+ + "version=" + version
+ + ", cluster=" + cluster
+ + '}';
+ }
+
+ public UpgradeableCluster getCluster()
+ {
+ return cluster;
+ }
+}
diff --git a/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/CassandraTestTemplate.java b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/CassandraTestTemplate.java
new file mode 100644
index 0000000..dcd8b3d
--- /dev/null
+++ b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/CassandraTestTemplate.java
@@ -0,0 +1,314 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.testing;
+
+import java.lang.reflect.AnnotatedElement;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Stream;
+
+import com.google.common.base.Preconditions;
+import org.junit.jupiter.api.extension.AfterTestExecutionCallback;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.Extension;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ParameterContext;
+import org.junit.jupiter.api.extension.ParameterResolver;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContext;
+import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.vdurmont.semver4j.Semver;
+import org.apache.cassandra.distributed.UpgradeableCluster;
+import org.apache.cassandra.distributed.api.Feature;
+import org.apache.cassandra.distributed.api.TokenSupplier;
+import org.apache.cassandra.distributed.shared.Versions;
+
+
+/**
+ * Creates a test per version of Cassandra we are testing
+ * Tests must be marked with {@link CassandraIntegrationTest}
+ * <p>
+ * This is a mix of parameterized tests + a custom extension. we need to be able to provide the test context
+ * to each test (like an extension) but also need to create multiple tests (like parameterized tests). Unfortunately
+ * the two don't play well with each other. You can't get access to the parameters from the extension.
+ * This test template allows us full control of the test lifecycle and lets us tightly couple the context to each test
+ * we generate, since the same test can be run for multiple versions of C*.
+ */
+public class CassandraTestTemplate implements TestTemplateInvocationContextProvider
+{
+ private static final Logger LOGGER = LoggerFactory.getLogger(CassandraTestTemplate.class);
+
+ private AbstractCassandraTestContext cassandraTestContext;
+
+ @Override
+ public boolean supportsTestTemplate(ExtensionContext context)
+ {
+ return true;
+ }
+
+ @Override
+ public Stream<TestTemplateInvocationContext> provideTestTemplateInvocationContexts(ExtensionContext context)
+ {
+ CassandraIntegrationTest annotation = getCassandraIntegrationTestAnnotation(context, true);
+ if (annotation.versionDependent())
+ {
+ return new TestVersionSupplier().testVersions()
+ .map(v -> invocationContext(v, context));
+ }
+ return Stream.of(invocationContext(new TestVersionSupplier().testVersions().findFirst().get(), context));
+ }
+
+ /**
+ * Returns a {@link TestTemplateInvocationContext}
+ *
+ * @param version a version for the test
+ * @param context the <em>context</em> in which the current test or container is being executed.
+ * @return the <em>context</em> of a single invocation of a
+ * {@linkplain org.junit.jupiter.api.TestTemplate test template}
+ */
+ private TestTemplateInvocationContext invocationContext(TestVersion version, ExtensionContext context)
+ {
+ return new CassandraTestTemplateInvocationContext(context, version);
+ }
+
+ private static CassandraIntegrationTest getCassandraIntegrationTestAnnotation(ExtensionContext context,
+ boolean throwIfNotFound)
+ {
+ Optional<AnnotatedElement> annotatedElement = context.getElement();
+ CassandraIntegrationTest result = annotatedElement.map(e -> e.getAnnotation(CassandraIntegrationTest.class))
+ .orElse(null);
+ if (result == null && throwIfNotFound)
+ {
+ throw new RuntimeException("CassandraTestTemplate could not "
+ + "find @CassandraIntegrationTest annotation");
+ }
+ return result;
+ }
+
+ private final class CassandraTestTemplateInvocationContext implements TestTemplateInvocationContext
+ {
+ private final ExtensionContext context;
+ private final TestVersion version;
+
+ private CassandraTestTemplateInvocationContext(ExtensionContext context, TestVersion version)
+ {
+ this.context = context;
+ this.version = version;
+ }
+
+ /**
+ * A display name can be configured per test still - this adds the C* version we're testing automatically
+ * as a suffix to the name
+ *
+ * @param invocationIndex the index to the invocation
+ * @return the display name
+ */
+ @Override
+ public String getDisplayName(int invocationIndex)
+ {
+ return context.getDisplayName() + ": " + version.version();
+ }
+
+ /**
+ * Used to register the extensions required to start and stop the in-jvm dtest environment
+ *
+ * @return a list of registered {@link Extension extensions}
+ */
+ @Override
+ public List<Extension> getAdditionalExtensions()
+ {
+ return Arrays.asList(parameterResolver(), postProcessor(), beforeEach());
+ }
+
+ private BeforeEachCallback beforeEach()
+ {
+ return beforeEachCtx -> {
+ CassandraIntegrationTest annotation = getCassandraIntegrationTestAnnotation(context, true);
+ // spin up a C* cluster using the in-jvm dtest
+ Versions versions = Versions.find();
+ int nodesPerDc = annotation.nodesPerDc();
+ int dcCount = annotation.numDcs();
+ int newNodesPerDc = annotation.newNodesPerDc(); // if the test wants to add more nodes later
+ Preconditions.checkArgument(newNodesPerDc >= 0,
+ "newNodesPerDc cannot be a negative number");
+ int originalNodeCount = nodesPerDc * dcCount;
+ int finalNodeCount = dcCount * (nodesPerDc + newNodesPerDc);
+ Versions.Version requestedVersion = versions.getLatest(new Semver(version.version(),
+ Semver.SemverType.LOOSE));
+ SimpleCassandraVersion versionParsed = SimpleCassandraVersion.create(version.version());
+
+ UpgradeableCluster.Builder clusterBuilder =
+ UpgradeableCluster.build(originalNodeCount)
+ // Disabling dynamic port allocaiton until we can get it working across JVM forks
+ //.withDynamicPortAllocation(true) // to allow parallel test runs
+ .withVersion(requestedVersion)
+ .withDCs(dcCount)
+ .withDataDirCount(annotation.numDataDirsPerInstance())
+ .withConfig(config -> annotationToFeatureList(annotation).forEach(config::with));
+ TokenSupplier tokenSupplier = TokenSupplier.evenlyDistributedTokens(finalNodeCount,
+ clusterBuilder.getTokenCount());
+ clusterBuilder.withTokenSupplier(tokenSupplier);
+ if (annotation.buildCluster())
+ {
+ UpgradeableCluster cluster;
+ cluster = clusterBuilder.createWithoutStarting();
+ if (annotation.startCluster())
+ {
+ cluster.startup();
+ }
+ cassandraTestContext = new CassandraTestContext(versionParsed, cluster, annotation);
+ }
+ else
+ {
+ cassandraTestContext = new ConfigurableCassandraTestContext(versionParsed,
+ clusterBuilder,
+ annotation);
+ }
+ LOGGER.info("Testing {} against in-jvm dtest cluster", version);
+ LOGGER.info("Created Cassandra test context {}", cassandraTestContext);
+ };
+ }
+
+ /**
+ * Shuts down the in-jvm dtest cluster when the test is finished
+ *
+ * @return the {@link AfterTestExecutionCallback}
+ */
+ private AfterTestExecutionCallback postProcessor()
+ {
+ return postProcessorCtx -> {
+ if (cassandraTestContext != null)
+ {
+ cassandraTestContext.close();
+ }
+ };
+ }
+
+ /**
+ * Builds a list of configured {@link Feature features} requested in the {@link CassandraIntegrationTest}
+ * annotation.
+ *
+ * @param annotation the configured annotation
+ * @return a list of configured {@link Feature features}
+ */
+ private List<Feature> annotationToFeatureList(CassandraIntegrationTest annotation)
+ {
+ List<Feature> configuredFeatures = new ArrayList<>();
+ if (annotation.nativeTransport())
+ {
+ configuredFeatures.add(Feature.NATIVE_PROTOCOL);
+ }
+ if (annotation.jmx())
+ {
+ configuredFeatures.add(Feature.JMX);
+ }
+ if (annotation.gossip())
+ {
+ configuredFeatures.add(Feature.GOSSIP);
+ }
+ if (annotation.network())
+ {
+ configuredFeatures.add(Feature.NETWORK);
+ }
+ return configuredFeatures;
+ }
+
+ /**
+ * Required for Junit to know the CassandraTestContext can be used in these tests
+ *
+ * @return a {@link ParameterResolver}
+ */
+ private ParameterResolver parameterResolver()
+ {
+ return new ParameterResolver()
+ {
+ @Override
+ public boolean supportsParameter(ParameterContext parameterContext,
+ ExtensionContext extensionContext)
+ {
+ Class<?> parameterType = parameterContext.getParameter().getType();
+ CassandraIntegrationTest annotation =
+ getCassandraIntegrationTestAnnotation(extensionContext, false);
+ if (annotation == null)
+ {
+ return false;
+ }
+ if (parameterType.equals(AbstractCassandraTestContext.class))
+ {
+ return true;
+ }
+ if (annotation.buildCluster())
+ {
+ if (parameterType.equals(CassandraTestContext.class))
+ {
+ return true;
+ }
+ else if (parameterType.equals(ConfigurableCassandraTestContext.class))
+ {
+ throw new IllegalArgumentException("CassandraIntegrationTest.buildCluster is true but"
+ + " a configurable context was requested. Please "
+ + "either request a CassandraTestContext "
+ + "as a parameter or set buildCluster to false");
+ }
+ }
+ else
+ {
+ if (parameterType.equals(ConfigurableCassandraTestContext.class))
+ {
+ return true;
+ }
+ else if (parameterType.equals(CassandraTestContext.class))
+ {
+ throw new IllegalArgumentException("CassandraIntegrationTest.buildCluster is false "
+ + "but a built cluster was requested. Please "
+ + "either request a "
+ + "ConfigurableCassandraTestContext as a "
+ + "parameter or set buildCluster to true"
+ + "(the default)");
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext)
+ {
+ return cassandraTestContext;
+ }
+ };
+ }
+ }
+
+ static
+ {
+ // Settings to reduce the test setup delay incurred if gossip is enabled
+ System.setProperty("cassandra.ring_delay_ms", "5000"); // down from 30s default
+ System.setProperty("cassandra.consistent.rangemovement", "false");
+ System.setProperty("cassandra.consistent.simultaneousmoves.allow", "true");
+ // End gossip delay settings
+ // Set the location of dtest jars
+ System.setProperty("cassandra.test.dtest_jar_path", System.getProperty("cassandra.test.dtest_jar_path", "dtest-jars"));
+ // Disable tcnative in netty as it can cause jni issues and logs lots errors
+ System.setProperty("cassandra.disable_tcactive_openssl", "true");
+ }
+}
diff --git a/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/ConfigurableCassandraTestContext.java b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/ConfigurableCassandraTestContext.java
new file mode 100644
index 0000000..9d4e0d5
--- /dev/null
+++ b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/ConfigurableCassandraTestContext.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.testing;
+
+import java.io.IOException;
+import java.util.function.Consumer;
+
+import org.apache.cassandra.distributed.UpgradeableCluster;
+
+/**
+ * A Cassandra Test Context implementation that allows advanced cluster configuration before cluster creation
+ * by providing access to the cluster builder.
+ */
+public class ConfigurableCassandraTestContext extends AbstractCassandraTestContext
+{
+ public static final String BUILT_CLUSTER_CANNOT_BE_CONFIGURED_ERROR =
+ "Cannot configure a cluster after it is built. Please set the buildCluster annotation attribute to false, "
+ + "and do not call `getCluster` before calling this method.";
+
+ private final UpgradeableCluster.Builder builder;
+
+ public ConfigurableCassandraTestContext(SimpleCassandraVersion version,
+ UpgradeableCluster.Builder builder,
+ CassandraIntegrationTest annotation)
+ {
+ super(version, annotation);
+ this.builder = builder;
+ }
+
+ public UpgradeableCluster configureCluster(Consumer<UpgradeableCluster.Builder> configurator) throws IOException
+ {
+ if (cluster != null)
+ {
+ throw new IllegalStateException(BUILT_CLUSTER_CANNOT_BE_CONFIGURED_ERROR);
+ }
+ configurator.accept(builder);
+ cluster = builder.createWithoutStarting();
+ return cluster;
+ }
+
+ public UpgradeableCluster configureAndStartCluster(Consumer<UpgradeableCluster.Builder> configurator)
+ throws IOException
+ {
+ cluster = configureCluster(configurator);
+ cluster.startup();
+ return cluster;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "ConfigurableCassandraTestContext{"
+ + ", version=" + version
+ + ", builder=" + builder
+ + '}';
+ }
+}
diff --git a/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/SimpleCassandraVersion.java b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/SimpleCassandraVersion.java
new file mode 100644
index 0000000..d7cd14b
--- /dev/null
+++ b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/SimpleCassandraVersion.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.testing;
+
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Implements versioning used in Cassandra and CQL.
+ * <p>
+ * Note: The following code uses a slight variation from the semver document (http://semver.org).
+ * </p>
+ * <p>
+ * The rules here are a bit different than normal semver comparison. For simplicity,
+ * an alpha version of 4.0 or a snapshot is equal to 4.0. This allows us to test sidecar
+ * against alpha versions of a release.
+ * <p>
+ * While it's possible to implement full version comparison, it's likely not very useful
+ * This is because the main testing we are going to do will be against release versions - something like 4.0.
+ * We want to list an adapter as being compatible with 4.0 - and that should include 4.0 alpha, etc.
+ */
+public class SimpleCassandraVersion implements Comparable<SimpleCassandraVersion>
+{
+ /**
+ * note: 3rd group matches to words but only allows number and checked after regexp test.
+ * this is because 3rd and the last can be identical.
+ **/
+ private static final String VERSION_REGEXP = "(\\d+)\\.(\\d+)(?:\\.(\\w+))?(\\-[.\\w]+)?([.+][.\\w]+)?";
+
+ private static final Pattern PATTERN = Pattern.compile(VERSION_REGEXP);
+ private static final String SNAPSHOT = "-SNAPSHOT";
+
+ public final int major;
+ public final int minor;
+ public final int patch;
+
+ /**
+ * Parse a version from a string.
+ *
+ * @param version the string to parse
+ * @return the {@link SimpleCassandraVersion} parsed from the {@code version} string
+ * @throws IllegalArgumentException if the provided string does not
+ * represent a version
+ */
+ public static SimpleCassandraVersion create(String version)
+ {
+ String stripped = version.toUpperCase().replace(SNAPSHOT, "");
+ Matcher matcher = PATTERN.matcher(stripped);
+ if (!matcher.matches())
+ {
+ throw new IllegalArgumentException("Invalid Cassandra version value: " + version);
+ }
+
+ try
+ {
+ int major = Integer.parseInt(matcher.group(1));
+ int minor = Integer.parseInt(matcher.group(2));
+ int patch = matcher.group(3) != null ? Integer.parseInt(matcher.group(3)) : 0;
+
+ return SimpleCassandraVersion.create(major, minor, patch);
+ }
+ catch (NumberFormatException e)
+ {
+ throw new IllegalArgumentException("Invalid Cassandra version value: " + version, e);
+ }
+ }
+
+ public static SimpleCassandraVersion create(int major, int minor, int patch)
+ {
+ if (major < 0 || minor < 0 || patch < 0)
+ {
+ throw new IllegalArgumentException();
+ }
+ return new SimpleCassandraVersion(major, minor, patch);
+ }
+
+
+ public SimpleCassandraVersion(int major, int minor, int patch)
+ {
+ this.major = major;
+ this.minor = minor;
+ this.patch = patch;
+ }
+
+
+ public int compareTo(SimpleCassandraVersion other)
+ {
+ if (major < other.major)
+ {
+ return -1;
+ }
+ if (major > other.major)
+ {
+ return 1;
+ }
+
+ if (minor < other.minor)
+ {
+ return -1;
+ }
+ if (minor > other.minor)
+ {
+ return 1;
+ }
+
+ if (patch < other.patch)
+ {
+ return -1;
+ }
+ if (patch > other.patch)
+ {
+ return 1;
+ }
+ return 0;
+ }
+
+ @Override
+ public boolean equals(Object o)
+ {
+ if (!(o instanceof SimpleCassandraVersion))
+ {
+ return false;
+ }
+ SimpleCassandraVersion that = (SimpleCassandraVersion) o;
+ return major == that.major
+ && minor == that.minor
+ && patch == that.patch;
+ }
+
+ /**
+ * Returns true if this > v2
+ *
+ * @param v2 the version to compare
+ * @return the result of the comparison
+ */
+ public boolean isGreaterThan(SimpleCassandraVersion v2)
+ {
+ return compareTo(v2) > 0;
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(major, minor, patch);
+ }
+
+ @Override
+ public String toString()
+ {
+ return major + "." + minor + "." + patch;
+ }
+}
diff --git a/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/TestVersion.java b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/TestVersion.java
new file mode 100644
index 0000000..7fb21f0
--- /dev/null
+++ b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/TestVersion.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.testing;
+
+/**
+ * Works with {@link TestVersionSupplier}
+ */
+public class TestVersion
+{
+ private final String version;
+
+ public TestVersion(String version)
+ {
+ this.version = version;
+ }
+
+ public String version()
+ {
+ return version;
+ }
+}
diff --git a/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/TestVersionSupplier.java b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/TestVersionSupplier.java
new file mode 100644
index 0000000..7295d51
--- /dev/null
+++ b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/TestVersionSupplier.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.testing;
+
+import java.util.Arrays;
+import java.util.stream.Stream;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Generates the list of versions we're going to test against.
+ * We will run the same module (trunk for example) against multiple versions of Cassandra.
+ * This is due to releases that don't add new features that would affect the sidecar,
+ * but we still want to test those versions specifically to avoid the chance of regressions.
+ *
+ * <p>Ideally, we'd probably have concurrent runs of the test infrastructure each running tests against one specific
+ * version of C*, but this will require some additional work in the dtest framework so for now we run one at a time.
+ */
+public class TestVersionSupplier
+{
+ private static final Logger LOGGER = LoggerFactory.getLogger(TestVersionSupplier.class);
+
+ Stream<TestVersion> testVersions()
+ {
+ // By default, we test 2 versions that will exercise oldest and newest supported versions
+ String versions = System.getProperty("cassandra.sidecar.versions_to_test", "4.0,4.1");
+ LOGGER.info("Testing with versions={}", versions);
+ return Arrays.stream(versions.split(",")).map(String::trim).map(TestVersion::new);
+ }
+}
diff --git a/cassandra-analytics-integration-tests/build.gradle b/cassandra-analytics-integration-tests/build.gradle
new file mode 100644
index 0000000..9d5efb4
--- /dev/null
+++ b/cassandra-analytics-integration-tests/build.gradle
@@ -0,0 +1,88 @@
+import java.nio.file.Paths
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+project(':cassandra-analytics-integration-tests') {
+ apply(plugin: 'java-library')
+ configurations {
+ configureEach {
+ resolutionStrategy {
+ force("com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}")
+ }
+ }
+ }
+ dependencies {
+
+ testImplementation(project(':cassandra-analytics-core'))
+ testImplementation(group: 'net.java.dev.jna', name: 'jna', version: '5.9.0')
+
+ testImplementation(group: "${sparkGroupId}", name: "spark-core_${scalaMajorVersion}", version: "${sparkVersion}")
+ testImplementation(group: "${sparkGroupId}", name: "spark-sql_${scalaMajorVersion}", version: "${sparkVersion}")
+
+ // JUnit
+ testImplementation("org.junit.jupiter:junit-jupiter-api:${project.junitVersion}")
+ testImplementation("org.junit.jupiter:junit-jupiter-params:${project.junitVersion}")
+ testImplementation("org.junit.jupiter:junit-jupiter-engine:${project.junitVersion}")
+ testImplementation(project(path: ":cassandra-analytics-integration-framework", configuration: "shadow"))
+ testImplementation(project(':cassandra-analytics-core-example'))
+ ext.dtestJar = System.getenv("DTEST_JAR") ?: "dtest-4.1.4.jar" // latest supported Cassandra build is 4.1
+ def dtestJarFullPath = "${dependencyLocation}${ext.dtestJar}"
+ println("Using DTest jar: " + dtestJarFullPath)
+ testImplementation(files("${dtestJarFullPath}"))
+ testImplementation("org.apache.cassandra:dtest-api:0.0.16")
+ // Needed by the Cassandra dtest framework
+ testImplementation("org.junit.vintage:junit-vintage-engine:${junitVersion}")
+ testImplementation('com.datastax.cassandra:cassandra-driver-core:3.9.0')
+
+ testImplementation('org.mockito:mockito-inline:4.10.0')
+ }
+
+
+ test {
+ // Because system properties aren't passed from the command line through to tests, we need to specifically
+ // set them again here.
+ systemProperty "cassandra.test.dtest_jar_path", dependencyLocation
+ systemProperty "cassandra.sidecar.versions_to_test", System.getProperty("cassandra.sidecar.versions_to_test", "4.0,4.1")
+ systemProperty "SKIP_STARTUP_VALIDATIONS", "true"
+ systemProperty "logback.configurationFile", "src/test/resources/logback-test.xml"
+ minHeapSize = '1024m'
+ maxHeapSize = '3072m'
+ // For now, keep maxParallelForks at 1 until we can fix dynamic port allocation across forks
+ maxParallelForks = 1
+ // maxParallelForks = Math.max(Runtime.runtime.availableProcessors() * 2, 8)
+ forkEvery = 1 // Enables different end-to-end test classes use Spark contexts with different configurations
+
+ // Make it so unit tests run on a JAr with Cassandra bridge implementations built in
+ dependsOn(tasks.jar)
+ classpath = project.sourceSets.test.output + configurations.testRuntimeClasspath + files(jar.archiveFile)
+ useJUnitPlatform()
+ def destDir = Paths.get(rootProject.rootDir.absolutePath, "build", "test-reports", "integration").toFile()
+ reports {
+ junitXml {
+ enabled true
+ destination = destDir
+ }
+ html{
+ enabled true
+ destination = destDir
+ }
+ }
+ }
+}
diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/IntegrationTestJob.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/IntegrationTestJob.java
new file mode 100644
index 0000000..40a03b4
--- /dev/null
+++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/IntegrationTestJob.java
@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cassandra.analytics;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.LongStream;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.spark.KryoRegister;
+import org.apache.cassandra.spark.bulkwriter.BulkSparkConf;
+import org.apache.cassandra.spark.bulkwriter.TTLOption;
+import org.apache.cassandra.spark.bulkwriter.TimestampOption;
+import org.apache.cassandra.spark.bulkwriter.WriterOptions;
+import org.apache.spark.SparkConf;
+import org.apache.spark.SparkContext;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.api.java.function.Function2;
+import org.apache.spark.sql.DataFrameWriter;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.RowFactory;
+import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.types.StructType;
+
+import static org.apache.spark.sql.types.DataTypes.BinaryType;
+import static org.apache.spark.sql.types.DataTypes.IntegerType;
+import static org.apache.spark.sql.types.DataTypes.LongType;
+
+/**
+ * Example showcasing the Cassandra Spark Analytics write and read capabilities
+ * <p>
+ * Prepare your environment by creating the following keyspace and table
+ * <p>
+ * Schema for the {@code keyspace}:
+ * <pre>
+ * CREATE KEYSPACE spark_test WITH replication = {'class': 'NetworkTopologyStrategy', 'datacenter1': '3'}
+ * AND durable_writes = true;
+ * </pre>
+ * <p>
+ * Schema for the {@code table}:
+ * <pre>
+ * CREATE TABLE spark_test.test (
+ * id BIGINT PRIMARY KEY,
+ * course BLOB,
+ * marks BIGINT
+ * );
+ * </pre>
+ */
+public final class IntegrationTestJob
+{
+ private static final Logger LOGGER = LoggerFactory.getLogger(IntegrationTestJob.class);
+ private static int sidecarPort;
+
+ private IntegrationTestJob()
+ {
+ throw new IllegalStateException(getClass() + " is static utility class and shall not be instantiated");
+ }
+
+ public static void main(String[] args)
+ {
+ if (args.length < 1)
+ {
+ sidecarPort = 9043;
+ }
+ else
+ {
+ sidecarPort = Integer.parseInt(args[0]);
+ }
+ LOGGER.info("Starting Spark job with args={}", Arrays.toString(args));
+
+ SparkConf sparkConf = new SparkConf().setAppName("Sample Spark Cassandra Bulk Reader Job")
+ .set("spark.master", "local[8]");
+
+ // Add SBW-specific settings
+ // TODO: Simplify setting up spark conf
+ BulkSparkConf.setupSparkConf(sparkConf, true);
+ KryoRegister.setup(sparkConf);
+
+ SparkSession spark = SparkSession
+ .builder()
+ .config(sparkConf)
+ .getOrCreate();
+ SparkContext sc = spark.sparkContext();
+ SQLContext sql = spark.sqlContext();
+ LOGGER.info("Spark Conf: " + sparkConf.toDebugString());
+ int rowCount = 10_000;
+
+ try
+ {
+ Dataset<Row> written = write(rowCount, sql, sc);
+ Dataset<Row> read = read(rowCount, sparkConf, sql, sc);
+ checkSmallDataFrameEquality(written, read);
+ LOGGER.info("Finished Spark job, shutting down...");
+ sc.stop();
+ }
+ catch (Throwable throwable)
+ {
+ LOGGER.error("Unexpected exception executing Spark job", throwable);
+ try
+ {
+ sc.stop();
+ }
+ catch (Throwable ignored)
+ {
+ }
+ throw throwable; // rethrow so the exception bubbles up to test usages.
+ }
+ }
+
+ private static Dataset<Row> write(long rowCount, SQLContext sql, SparkContext sc)
+ {
+ JavaSparkContext javaSparkContext = JavaSparkContext.fromSparkContext(sc);
+ int parallelism = sc.defaultParallelism();
+ boolean addTTLColumn = false;
+ boolean addTimestampColumn = false;
+ JavaRDD<Row> rows = genDataset(javaSparkContext, rowCount, parallelism, addTTLColumn, addTimestampColumn);
+ Dataset<Row> df = sql.createDataFrame(rows, getWriteSchema(addTTLColumn, addTimestampColumn));
+
+ DataFrameWriter<Row> dfWriter = df.write()
+ .format("org.apache.cassandra.spark.sparksql.CassandraDataSink")
+ .option("sidecar_instances", "localhost,localhost2,localhost3")
+ .option("sidecar_port", sidecarPort)
+ .option("keyspace", "spark_test")
+ .option("table", "test")
+ .option("local_dc", "datacenter1")
+ .option("bulk_writer_cl", "LOCAL_QUORUM")
+ .option("number_splits", "-1")
+ // A constant timestamp and TTL can be used by setting the following options.
+ // .option(WriterOptions.TTL.name(), TTLOption.constant(20))
+ // .option(WriterOptions.TIMESTAMP.name(), TimestampOption.constant(System.currentTimeMillis() * 1000))
+ .mode("append");
+
+ List<String> addedColumns = new ArrayList<>();
+ if (addTTLColumn)
+ {
+ addedColumns.add("ttl");
+ dfWriter = dfWriter
+ .option(WriterOptions.TTL.name(), TTLOption.perRow("ttl"));
+ }
+
+ if (addTimestampColumn)
+ {
+ addedColumns.add("timestamp");
+ dfWriter = dfWriter
+ .option(WriterOptions.TIMESTAMP.name(), TimestampOption.perRow("timestamp"));
+ }
+
+ dfWriter.save();
+
+ if (!addedColumns.isEmpty())
+ {
+ df = df.drop(addedColumns.toArray(new String[0]));
+ }
+
+ return df;
+ }
+
+ private static Dataset<Row> read(int expectedRowCount, SparkConf sparkConf, SQLContext sql, SparkContext sc)
+ {
+ int coresPerExecutor = sparkConf.getInt("spark.executor.cores", 1);
+ int numExecutors = sparkConf.getInt("spark.dynamicAllocation.maxExecutors", sparkConf.getInt("spark.executor.instances", 1));
+ int numCores = coresPerExecutor * numExecutors;
+
+ Dataset<Row> df = sql.read().format("org.apache.cassandra.spark.sparksql.CassandraDataSource")
+ .option("sidecar_instances", "localhost,localhost2,localhost3")
+ .option("sidecar_port", sidecarPort)
+ .option("keyspace", "spark_test")
+ .option("table", "test")
+ .option("DC", "datacenter1")
+ .option("snapshotName", UUID.randomUUID().toString())
+ .option("createSnapshot", "true")
+ // Shutdown hooks are called after the job ends, and in the case of integration tests
+ // the sidecar is already shut down before this. Since the cluster will be torn
+ // down anyway, the integration job skips clearing snapshots.
+ .option("clearSnapshot", "false")
+ .option("defaultParallelism", sc.defaultParallelism())
+ .option("numCores", numCores)
+ .option("sizing", "default")
+ .load();
+
+ long count = df.count();
+ LOGGER.info("Found {} records", count);
+
+ if (count != expectedRowCount)
+ {
+ throw new RuntimeException(String.format("Expected %d records but found %d records",
+ expectedRowCount,
+ count));
+ }
+ return df;
+ }
+
+ private static StructType getWriteSchema(boolean addTTLColumn, boolean addTimestampColumn)
+ {
+ StructType schema = new StructType()
+ .add("id", LongType, false)
+ .add("course", BinaryType, false)
+ .add("marks", LongType, false);
+ if (addTTLColumn)
+ {
+ schema = schema.add("ttl", IntegerType, false);
+ }
+ if (addTimestampColumn)
+ {
+ schema = schema.add("timestamp", LongType, false);
+ }
+ return schema;
+ }
+
+ private static void checkSmallDataFrameEquality(Dataset<Row> expected, Dataset<Row> actual)
+ {
+ if (actual == null)
+ {
+ throw new NullPointerException("actual dataframe is null");
+ }
+ if (expected == null)
+ {
+ throw new NullPointerException("expected dataframe is null");
+ }
+ // Simulate `actual` having fewer rows, but all match rows in `expected`.
+ // The previous implementation would consider these equal
+ // actual = actual.limit(1000);
+ if (!actual.exceptAll(expected).isEmpty() || !expected.exceptAll(actual).isEmpty())
+ {
+ throw new IllegalStateException("The content of the dataframes differs");
+ }
+ }
+
+ private static JavaRDD<Row> genDataset(JavaSparkContext sc, long records, Integer parallelism,
+ boolean addTTLColumn, boolean addTimestampColumn)
+ {
+ long recordsPerPartition = records / parallelism;
+ long remainder = records - (recordsPerPartition * parallelism);
+ List<Integer> seq = IntStream.range(0, parallelism).boxed().collect(Collectors.toList());
+ int ttl = 120; // data will not be queryable in two minutes
+ long timeStamp = System.currentTimeMillis() * 1000;
+ JavaRDD<Row> dataset = sc.parallelize(seq, parallelism).mapPartitionsWithIndex(
+ (Function2<Integer, Iterator<Integer>, Iterator<Row>>) (index, integerIterator) -> {
+ long firstRecordNumber = index * recordsPerPartition;
+ long recordsToGenerate = index.equals(parallelism) ? remainder : recordsPerPartition;
+ java.util.Iterator<Row> rows = LongStream.range(0, recordsToGenerate).mapToObj(offset -> {
+ long recordNumber = firstRecordNumber + offset;
+ String courseNameString = String.valueOf(recordNumber);
+ Integer courseNameStringLen = courseNameString.length();
+ Integer courseNameMultiplier = 1000 / courseNameStringLen;
+ byte[] courseName = dupStringAsBytes(courseNameString, courseNameMultiplier);
+ ArrayList<Object> values = new ArrayList<>(Arrays.asList(recordNumber, courseName, recordNumber));
+ if (addTTLColumn)
+ {
+ values.add(ttl);
+ }
+ if (addTimestampColumn)
+ {
+ values.add(timeStamp);
+ }
+ return RowFactory.create(values.toArray());
+ }).iterator();
+ return rows;
+ }, false);
+ return dataset;
+ }
+
+ private static byte[] dupStringAsBytes(String string, Integer times)
+ {
+ byte[] stringBytes = string.getBytes();
+ ByteBuffer buffer = ByteBuffer.allocate(stringBytes.length * times);
+ for (int time = 0; time < times; time++)
+ {
+ buffer.put(stringBytes);
+ }
+ return buffer.array();
+ }
+}
diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/SparkBulkWriterSimpleTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/SparkBulkWriterSimpleTest.java
new file mode 100644
index 0000000..cb2063c
--- /dev/null
+++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/SparkBulkWriterSimpleTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cassandra.analytics;
+
+import org.apache.cassandra.distributed.UpgradeableCluster;
+import org.apache.cassandra.sidecar.testing.IntegrationTestBase;
+import org.apache.cassandra.testing.CassandraIntegrationTest;
+
+public class SparkBulkWriterSimpleTest extends IntegrationTestBase
+{
+ @CassandraIntegrationTest(nodesPerDc = 3)
+ public void runSampleJob() throws InterruptedException
+ {
+ UpgradeableCluster cluster = sidecarTestContext.cluster();
+ String keyspace = "spark_test";
+ String table = "test";
+ cluster.schemaChange(
+ " CREATE KEYSPACE " + keyspace + " WITH replication = "
+ + "{'class': 'NetworkTopologyStrategy', 'datacenter1': '3'}\n"
+ + " AND durable_writes = true;");
+ cluster.schemaChange("CREATE TABLE " + keyspace + "." + table + " (\n"
+ + " id BIGINT PRIMARY KEY,\n"
+ + " course BLOB,\n"
+ + " marks BIGINT\n"
+ + " );");
+ waitForKeyspaceAndTable(keyspace, table);
+ IntegrationTestJob.main(new String[]{String.valueOf(server.actualPort())});
+ }
+}
diff --git a/cassandra-analytics-integration-tests/src/test/resources/logback-test.xml b/cassandra-analytics-integration-tests/src/test/resources/logback-test.xml
new file mode 100644
index 0000000..d1b7318
--- /dev/null
+++ b/cassandra-analytics-integration-tests/src/test/resources/logback-test.xml
@@ -0,0 +1,48 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>WARN</level>
+ </filter>
+ <encoder>
+ <pattern>%-5level [%thread] %date{ISO8601} %F:%L - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="FILE" class="ch.qos.logback.core.FileAppender">
+ <file>${LOG_FILE_PATH:-../build/test-reports/complete-test-output.log}</file>
+ <encoder>
+ <pattern>%-5level [%thread] %date{ISO8601} %msg%n</pattern>
+ </encoder>
+ <immediateFlush>true</immediateFlush>
+ </appender>
+
+ <root level="INFO">
+ <appender-ref ref="STDOUT" />
+ </root>
+ <root level="DEBUG">
+ <appender-ref ref="FILE" />
+ </root>
+
+
+ <logger name="com.datastax.driver.core" level="ERROR" />
+ <logger name="com.datastax.driver.core.ControlConnection" level="OFF" />
+</configuration>
diff --git a/cassandra-bridge/src/main/java/org/apache/cassandra/bridge/CassandraVersion.java b/cassandra-bridge/src/main/java/org/apache/cassandra/bridge/CassandraVersion.java
index e8bb5c3..e7343c6 100644
--- a/cassandra-bridge/src/main/java/org/apache/cassandra/bridge/CassandraVersion.java
+++ b/cassandra-bridge/src/main/java/org/apache/cassandra/bridge/CassandraVersion.java
@@ -30,7 +30,6 @@ public enum CassandraVersion
THREEZERO(30, "3.0", "three-zero"),
FOURZERO(40, "4.0", "four-zero"),
FOURONE(41, "4.1", "four-zero");
-
private final int number;
private final String name;
private final String jarBaseName; // Must match shadowJar.archiveFileName from Gradle configuration (without extension)
diff --git a/cassandra-four-zero/build.gradle b/cassandra-four-zero/build.gradle
index cee96b7..b93e8fb 100644
--- a/cassandra-four-zero/build.gradle
+++ b/cassandra-four-zero/build.gradle
@@ -1,3 +1,5 @@
+import java.nio.file.Paths
+
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -84,12 +86,23 @@ project(':cassandra-four-zero') {
tasks.register('javadocJar', Jar) {
dependsOn javadoc
- classifier = 'javadoc'
+ archiveClassifier.set('javadoc')
from javadoc.destinationDir
}
test {
useJUnitPlatform()
+ reports {
+ def destDir = Paths.get(rootProject.rootDir.absolutePath, "build", "test-reports", "four-zero").toFile()
+ junitXml {
+ enabled true
+ destination = destDir
+ }
+ html{
+ enabled true
+ destination = destDir
+ }
+ }
}
artifacts {
diff --git a/config/checkstyle/checkstyle.xml b/config/checkstyle/checkstyle.xml
index 39a53e9..49c4377 100644
--- a/config/checkstyle/checkstyle.xml
+++ b/config/checkstyle/checkstyle.xml
@@ -112,7 +112,7 @@
<!-- Checks for naming conventions. -->
<!-- See https://checkstyle.org/config_naming.html -->
- <module name="ConstantName" />
+<!-- <module name="ConstantName" />-->
<module name="LocalFinalVariableName" />
<module name="LocalVariableName" />
<module name="MemberName" />
@@ -146,7 +146,7 @@
<module name="MethodParamPad" />
<module name="NoWhitespaceAfter" />
<module name="NoWhitespaceBefore" />
- <module name="OperatorWrap" />
+<!-- <module name="OperatorWrap" />-->
<module name="ParenPad" />
<module name="TypecastParenPad" />
<module name="WhitespaceAfter" />
@@ -203,16 +203,9 @@
<!-- Checks for class design. -->
<!-- See https://checkstyle.org/config_design.html -->
<module name="DesignForExtension" />
- <module name="FinalClass" />
+<!-- <module name="FinalClass" />-->
<module name="HideUtilityClassConstructor" />
<module name="InterfaceIsType" />
- <module name="VisibilityModifier">
- <property name="packageAllowed" value="true" />
- <property name="protectedAllowed" value="true" />
- <property name="allowPublicFinalFields" value="true" />
- <property name="allowPublicImmutableFields" value="true" />
- </module>
-
<!-- Miscellaneous other checks. -->
<!-- See https://checkstyle.org/config_misc.html -->
<module name="ArrayTypeStyle" />
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
index cc4fdc2..7454180 100644
Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index c582b66..db9a6b8 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -1,22 +1,5 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
diff --git a/gradlew b/gradlew
index eb0e1a4..1b6c787 100755
--- a/gradlew
+++ b/gradlew
@@ -1,78 +1,129 @@
-#!/usr/bin/env sh
+#!/bin/sh
+
+#
+# Copyright © 2015-2021 the original authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
##############################################################################
-##
-## Gradle start up script for UN*X
-##
+#
+# Gradle start up script for POSIX generated by Gradle.
+#
+# Important for running:
+#
+# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
+# noncompliant, but you have some other compliant shell such as ksh or
+# bash, then to run this script, type that shell name before the whole
+# command line, like:
+#
+# ksh Gradle
+#
+# Busybox and similar reduced shells will NOT work, because this script
+# requires all of these POSIX shell features:
+# * functions;
+# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
+# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
+# * compound commands having a testable exit status, especially «case»;
+# * various built-in commands including «command», «set», and «ulimit».
+#
+# Important for patching:
+#
+# (2) This script targets any POSIX shell, so it avoids extensions provided
+# by Bash, Ksh, etc; in particular arrays are avoided.
+#
+# The "traditional" practice of packing multiple parameters into a
+# space-separated string is a well documented source of bugs and security
+# problems, so this is (mostly) avoided, by progressively accumulating
+# options in "$@", and eventually passing that to Java.
+#
+# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
+# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
+# see the in-line comments for details.
+#
+# There are tweaks for specific operating systems such as AIX, CygWin,
+# Darwin, MinGW, and NonStop.
+#
+# (3) This script is generated from the Groovy template
+# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+# within the Gradle project.
+#
+# You can find Gradle at https://github.com/gradle/gradle/.
+#
##############################################################################
# Attempt to set APP_HOME
+
# Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
- ls=`ls -ld "$PRG"`
- link=`expr "$ls" : '.*-> \(.*\)$'`
- if expr "$link" : '/.*' > /dev/null; then
- PRG="$link"
- else
- PRG=`dirname "$PRG"`"/$link"
- fi
+app_path=$0
+
+# Need this for daisy-chained symlinks.
+while
+ APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
+ [ -h "$app_path" ]
+do
+ ls=$( ls -ld "$app_path" )
+ link=${ls#*' -> '}
+ case $link in #(
+ /*) app_path=$link ;; #(
+ *) app_path=$APP_HOME$link ;;
+ esac
done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >/dev/null
-APP_HOME="`pwd -P`"
-cd "$SAVED" >/dev/null
+
+APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
APP_NAME="Gradle"
-APP_BASE_NAME=`basename "$0"`
+APP_BASE_NAME=${0##*/}
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Use the maximum available, or set MAX_FD != -1 to use that value.
-MAX_FD="maximum"
+MAX_FD=maximum
warn () {
echo "$*"
-}
+} >&2
die () {
echo
echo "$*"
echo
exit 1
-}
+} >&2
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
nonstop=false
-case "`uname`" in
- CYGWIN* )
- cygwin=true
- ;;
- Darwin* )
- darwin=true
- ;;
- MINGW* )
- msys=true
- ;;
- NONSTOP* )
- nonstop=true
- ;;
+case "$( uname )" in #(
+ CYGWIN* ) cygwin=true ;; #(
+ Darwin* ) darwin=true ;; #(
+ MSYS* | MINGW* ) msys=true ;; #(
+ NONSTOP* ) nonstop=true ;;
esac
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
- JAVACMD="$JAVA_HOME/jre/sh/java"
+ JAVACMD=$JAVA_HOME/jre/sh/java
else
- JAVACMD="$JAVA_HOME/bin/java"
+ JAVACMD=$JAVA_HOME/bin/java
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
@@ -81,7 +132,7 @@ Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
- JAVACMD="java"
+ JAVACMD=java
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
@@ -89,79 +140,95 @@ location of your Java installation."
fi
# Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
- MAX_FD_LIMIT=`ulimit -H -n`
- if [ $? -eq 0 ] ; then
- if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
- MAX_FD="$MAX_FD_LIMIT"
- fi
- ulimit -n $MAX_FD
- if [ $? -ne 0 ] ; then
- warn "Could not set maximum file descriptor limit: $MAX_FD"
- fi
- else
- warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
- fi
+if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
+ case $MAX_FD in #(
+ max*)
+ MAX_FD=$( ulimit -H -n ) ||
+ warn "Could not query maximum file descriptor limit"
+ esac
+ case $MAX_FD in #(
+ '' | soft) :;; #(
+ *)
+ ulimit -n "$MAX_FD" ||
+ warn "Could not set maximum file descriptor limit to $MAX_FD"
+ esac
fi
-# For Darwin, add options to specify how the application appears in the dock
-if $darwin; then
- GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
-fi
+# Collect all arguments for the java command, stacking in reverse order:
+# * args from the command line
+# * the main class name
+# * -classpath
+# * -D...appname settings
+# * --module-path (only if needed)
+# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
# For Cygwin or MSYS, switch paths to Windows format before running java
-if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
- APP_HOME=`cygpath --path --mixed "$APP_HOME"`
- CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
- JAVACMD=`cygpath --unix "$JAVACMD"`
-
- # We build the pattern for arguments to be converted via cygpath
- ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
- SEP=""
- for dir in $ROOTDIRSRAW ; do
- ROOTDIRS="$ROOTDIRS$SEP$dir"
- SEP="|"
- done
- OURCYGPATTERN="(^($ROOTDIRS))"
- # Add a user-defined pattern to the cygpath arguments
- if [ "$GRADLE_CYGPATTERN" != "" ] ; then
- OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
- fi
+if "$cygwin" || "$msys" ; then
+ APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
+ CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
+
+ JAVACMD=$( cygpath --unix "$JAVACMD" )
+
# Now convert the arguments - kludge to limit ourselves to /bin/sh
- i=0
- for arg in "$@" ; do
- CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
- CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
-
- if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
- eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
- else
- eval `echo args$i`="\"$arg\""
+ for arg do
+ if
+ case $arg in #(
+ -*) false ;; # don't mess with options #(
+ /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
+ [ -e "$t" ] ;; #(
+ *) false ;;
+ esac
+ then
+ arg=$( cygpath --path --ignore --mixed "$arg" )
fi
- i=`expr $i + 1`
+ # Roll the args list around exactly as many times as the number of
+ # args, so each arg winds up back in the position where it started, but
+ # possibly modified.
+ #
+ # NB: a `for` loop captures its iteration list before it begins, so
+ # changing the positional parameters here affects neither the number of
+ # iterations, nor the values presented in `arg`.
+ shift # remove old arg
+ set -- "$@" "$arg" # push replacement arg
done
- case $i in
- 0) set -- ;;
- 1) set -- "$args0" ;;
- 2) set -- "$args0" "$args1" ;;
- 3) set -- "$args0" "$args1" "$args2" ;;
- 4) set -- "$args0" "$args1" "$args2" "$args3" ;;
- 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
- 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
- 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
- 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
- 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
- esac
fi
-# Escape application args
-save () {
- for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
- echo " "
-}
-APP_ARGS=$(save "$@")
-
-# Collect all arguments for the java command, following the shell quoting and substitution rules
-eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+# Collect all arguments for the java command;
+# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
+# shell script including quotes and variable substitutions, so put them in
+# double quotes to make sure that they get re-expanded; and
+# * put everything else in single quotes, so that it's not re-expanded.
+
+set -- \
+ "-Dorg.gradle.appname=$APP_BASE_NAME" \
+ -classpath "$CLASSPATH" \
+ org.gradle.wrapper.GradleWrapperMain \
+ "$@"
+
+# Use "xargs" to parse quoted args.
+#
+# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
+#
+# In Bash we could simply go:
+#
+# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
+# set -- "${ARGS[@]}" "$@"
+#
+# but POSIX shell has neither arrays nor command substitution, so instead we
+# post-process each arg (as a line of input to sed) to backslash-escape any
+# character that might be a shell metacharacter, then use eval to reverse
+# that process (while maintaining the separation between arguments), and wrap
+# the whole thing up as a single "set" statement.
+#
+# This will of course break if any of these variables contains a newline or
+# an unmatched quote.
+#
+
+eval "set -- $(
+ printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
+ xargs -n1 |
+ sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
+ tr '\n' ' '
+ )" '"$@"'
exec "$JAVACMD" "$@"
diff --git a/profiles/scala-2.11-spark-2-jdk-1.8.gradle b/profiles/scala-2.11-spark-2-jdk-1.8.gradle
index 83db4ad..4f22ab9 100644
--- a/profiles/scala-2.11-spark-2-jdk-1.8.gradle
+++ b/profiles/scala-2.11-spark-2-jdk-1.8.gradle
@@ -18,7 +18,8 @@
*/
ext {
- jacksonVersion="2.6.7.5"
+ jacksonVersion="2.6.7"
+ jacksonScalaModuleVersion="2.6.7.1"
scalaMajorVersion="2.11"
scalaVersion="2.11.12"
sparkGroupId="org.apache.spark"
@@ -26,5 +27,5 @@ ext {
sparkVersion="2.4.8"
sidecarClientGroup="org.apache.cassandra.sidecar"
sidecarClientName="vertx-client-all"
- sidecarVersion="1.0.0-jdk8-local"
+ sidecarVersion="1.0.0-jdk8-analytics"
}
diff --git a/profiles/scala-2.12-spark-2-jdk-1.8.gradle b/profiles/scala-2.12-spark-2-jdk-1.8.gradle
index 3923a1e..373309b 100644
--- a/profiles/scala-2.12-spark-2-jdk-1.8.gradle
+++ b/profiles/scala-2.12-spark-2-jdk-1.8.gradle
@@ -18,7 +18,8 @@
*/
ext {
- jacksonVersion="2.6.7.3"
+ jacksonVersion="2.6.7"
+ jacksonScalaModuleVersion="2.6.7.1"
scalaMajorVersion="2.12"
scalaVersion="2.12.10"
sparkGroupId="org.apache.spark"
@@ -26,5 +27,5 @@ ext {
sparkVersion="2.4.8"
sidecarClientGroup="org.apache.cassandra.sidecar"
sidecarClientName="vertx-client-all"
- sidecarVersion="1.0.0-jdk8-local"
+ sidecarVersion="1.0.0-jdk8-analytics"
}
diff --git a/profiles/scala-2.12-spark-3-jdk-11.gradle b/profiles/scala-2.12-spark-3-jdk-11.gradle
index 8e3cfc5..a7317c3 100644
--- a/profiles/scala-2.12-spark-3-jdk-11.gradle
+++ b/profiles/scala-2.12-spark-3-jdk-11.gradle
@@ -19,6 +19,7 @@
ext {
jacksonVersion="2.14.2"
+ jacksonScalaModuleVersion="2.14.2"
scalaMajorVersion="2.12"
scalaVersion="2.12.10"
sparkGroupId="org.apache.spark"
@@ -26,5 +27,5 @@ ext {
sparkVersion="3.2.2"
sidecarClientGroup="org.apache.cassandra.sidecar"
sidecarClientName="vertx-client-all"
- sidecarVersion="1.0.0-local"
+ sidecarVersion="1.0.0-analytics"
}
diff --git a/profiles/scala-2.13-spark-3-jdk-11.gradle b/profiles/scala-2.13-spark-3-jdk-11.gradle
index f5f9f5b..4d2ba70 100644
--- a/profiles/scala-2.13-spark-3-jdk-11.gradle
+++ b/profiles/scala-2.13-spark-3-jdk-11.gradle
@@ -19,6 +19,7 @@
ext {
jacksonVersion="2.12.7"
+ jacksonScalaModuleVersion="2.12.7"
scalaMajorVersion="2.13"
scalaVersion="2.13.10"
sparkGroupId="org.apache.spark"
@@ -26,5 +27,5 @@ ext {
sparkVersion="3.2.2"
sidecarClientGroup="org.apache.cassandra.sidecar"
sidecarClientName="vertx-client-all"
- sidecarVersion="1.0.0-local"
+ sidecarVersion="1.0.0-analytics"
}
diff --git a/scripts/.mvn/wrapper/maven-wrapper.jar b/scripts/.mvn/wrapper/maven-wrapper.jar
new file mode 100644
index 0000000..cb28b0e
Binary files /dev/null and b/scripts/.mvn/wrapper/maven-wrapper.jar differ
diff --git a/gradle/wrapper/gradle-wrapper.properties b/scripts/.mvn/wrapper/maven-wrapper.properties
similarity index 77%
copy from gradle/wrapper/gradle-wrapper.properties
copy to scripts/.mvn/wrapper/maven-wrapper.properties
index c582b66..70f4f50 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/scripts/.mvn/wrapper/maven-wrapper.properties
@@ -14,9 +14,5 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-distributionBase=GRADLE_USER_HOME
-distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip
-zipStoreBase=GRADLE_USER_HOME
-zipStorePath=wrapper/dists
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.8/apache-maven-3.8.8-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar
diff --git a/scripts/build-dependencies.sh b/scripts/build-dependencies.sh
new file mode 100755
index 0000000..e6514ea
--- /dev/null
+++ b/scripts/build-dependencies.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+SCRIPT_DIR=$( dirname -- "$( readlink -f -- "$0"; )"; )
+
+${SCRIPT_DIR}/build-dtest-jars.sh
+${SCRIPT_DIR}/build-sidecar.sh
\ No newline at end of file
diff --git a/scripts/build-dtest-jars.sh b/scripts/build-dtest-jars.sh
new file mode 100755
index 0000000..40e461a
--- /dev/null
+++ b/scripts/build-dtest-jars.sh
@@ -0,0 +1,121 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ set -xe
+
+if [ "${SKIP_DTEST_JAR_BUILD}" == "true" ]; then
+ echo "Skipping DTest Jar Build because SKIP_DTEST_JAR_BUILD was set to 'true'"
+else
+ # We use a mapping from branch -> `commit-ish` here in case something committed
+ # to one of the Cassandra branches breaks us. By default, these should be release tags
+ # unless there is a particular commit that is needed but not yet released,
+ # or a breaking change in a released branch (which should be fixed with some urgency).
+ # If replacing a release tag with a SHA, please raise a JIRA for the issue identified
+ # and add a comment here explaining why the SHA was picked, and what the condition for
+ # moving back to a release tag would be.
+ # Examples
+ # a tagged release of Cassandra 4.0
+ # "cassandra-4.0:cassandra-4.0.12"
+ # a hash that points to a commit on the cassandra-4.0 branch
+ # "cassandra-4.0:1f79c8492528f01bcc5f88951a1cc9e0d7265c54"
+ # the cassandra-4.0 branch - used for nightly integration test runs or local testing of new features
+ # "cassandra-4.0:cassandra-4.0"
+ # Due to MacOS being stuck on Bash < 4, we don't use associative arrays here.
+ CANDIDATE_BRANCHES=(
+ "cassandra-4.0:1f79c8492528f01bcc5f88951a1cc9e0d7265c54"
+ "cassandra-4.1:725655dda2776fef35567496a6e331102eb7610d"
+ )
+ BRANCHES=( ${BRANCHES:-cassandra-4.0 cassandra-4.1} )
+ echo ${BRANCHES[*]}
+ REPO=${REPO:-"https://github.com/apache/cassandra.git"}
+ SCRIPT_DIR=$( dirname -- "$( readlink -f -- "$0"; )"; )
+ DTEST_JAR_DIR="$(dirname "${SCRIPT_DIR}/")/dependencies"
+ DTEST_JAR_DIR=${CASSANDRA_DEP_DIR:-$DTEST_JAR_DIR}
+ BUILD_DIR="${DTEST_JAR_DIR}/cassandra-build"
+
+ if [[ "$CLEAN" == "true" ]]; then
+ echo "Clean up $DTEST_JAR_DIR"
+ rm -rf "$DTEST_JAR_DIR/cassandra-build"
+ rm "$DTEST_JAR_DIR/dtest*.jar"
+ fi
+
+ source "$SCRIPT_DIR/functions.sh"
+ mkdir -p "${BUILD_DIR}"
+
+ # host key verification
+ mkdir -p ~/.ssh
+ REPO_HOST=$(get_hostname "${REPO}")
+ ssh-keyscan "${REPO_HOST}" >> ~/.ssh/known_hosts || true
+
+ for index in "${!CANDIDATE_BRANCHES[@]}"; do
+ cd "${BUILD_DIR}"
+ branchSha=(${CANDIDATE_BRANCHES[$index]//:/ })
+ branch=${branchSha[0]}
+ sha=${branchSha[1]}
+
+ if ! [[ "${BRANCHES[@]}" =~ "$branch" ]]; then
+ echo "branch ${branch} is not selected to build. The selected branches are ${BRANCHES[*]}"
+ continue
+ fi
+
+ echo "index ${index} branch ${branch} sha ${sha}"
+ # check out the correct cassandra version:
+ if [ ! -d "${branch}" ] ; then
+ if [ -n "${sha}" ] ; then
+ mkdir -p "${branch}"
+ cd "${branch}"
+ git init
+ git remote add upstream "${REPO}"
+ git fetch --depth=1 upstream "${sha}"
+ git reset --hard FETCH_HEAD
+ else
+ git clone --depth 1 --single-branch --branch "${branch}" "${REPO}" "${branch}"
+ cd "${branch}"
+ fi
+ else
+ cd "${branch}"
+ if [ -z "${sha}" ] ; then
+ git pull
+ fi
+ fi
+ if [ -z "${sha}" ] ; then
+ git checkout "${branch}"
+ fi
+ git clean -fd
+ CASSANDRA_VERSION=$(cat build.xml | grep 'property name="base.version"' | awk -F "\"" '{print $4}')
+ # Loop to prevent failure due to maven-ant-tasks not downloading a jar.
+ for x in $(seq 1 3); do
+ if [ -f "${DTEST_JAR_DIR}/dtest-${CASSANDRA_VERSION}.jar" ]; then
+ RETURN="0"
+ break
+ else
+ "${SCRIPT_DIR}/build-shaded-dtest-jar-local.sh"
+ RETURN="$?"
+ if [ "${RETURN}" -eq "0" ]; then
+ break
+ fi
+ fi
+ done
+ # Exit, if we didn't build successfully
+ if [ "${RETURN}" -ne "0" ]; then
+ echo "Build failed with exit code: ${RETURN}"
+ exit ${RETURN}
+ fi
+ done
+fi
diff --git a/scripts/build-shaded-dtest-jar-local.sh b/scripts/build-shaded-dtest-jar-local.sh
new file mode 100755
index 0000000..79b39bb
--- /dev/null
+++ b/scripts/build-shaded-dtest-jar-local.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -xe
+
+ARTIFACT_NAME=cassandra-dtest
+REPO_DIR="$(pwd)/out"
+SCRIPT_DIR=$( dirname -- "$( readlink -f -- "$0"; )"; )
+CASSANDRA_VERSION=$(cat build.xml | grep 'property name="base.version"' | awk -F "\"" '{print $4}')
+GIT_HASH=$(git rev-parse --short HEAD)
+DTEST_ARTIFACT_ID=${ARTIFACT_NAME}-local
+DTEST_JAR_DIR="$(dirname "${SCRIPT_DIR}/")/dependencies"
+DTEST_JAR_DIR=${CASSANDRA_DEP_DIR:-$DTEST_JAR_DIR}
+
+echo "${CASSANDRA_VERSION}"
+echo "${GIT_HASH}"
+echo "${DTEST_ARTIFACT_ID}"
+
+ant realclean
+ant dtest-jar -Dno-checkstyle=true
+
+# Install the version that will be shaded
+"${SCRIPT_DIR}/mvnw" install:install-file \
+ -Dfile="./build/dtest-${CASSANDRA_VERSION}.jar" \
+ -DgroupId=org.apache.cassandra \
+ -DartifactId="${DTEST_ARTIFACT_ID}" \
+ -Dversion="${CASSANDRA_VERSION}-${GIT_HASH}" \
+ -Dpackaging=jar \
+ -DgeneratePom=true \
+ -DlocalRepositoryPath="${REPO_DIR}"
+
+# Create shaded artifact
+"${SCRIPT_DIR}/mvnw" --file "${SCRIPT_DIR}/relocate-dtest-dependencies.pom" package \
+ -Drevision="${CASSANDRA_VERSION}" \
+ -DskipTests \
+ -Ddtest.version="${CASSANDRA_VERSION}-${GIT_HASH}" \
+ -Dmaven.repo.local="${REPO_DIR}" \
+ -DoutputFilePath="${DTEST_JAR_DIR}/dtest-${CASSANDRA_VERSION}.jar" \
+ -Drelocation.prefix="shaded-${GIT_HASH}" \
+ --no-snapshot-updates --update-snapshots
+
+set +xe
diff --git a/scripts/build-sidecar.sh b/scripts/build-sidecar.sh
new file mode 100755
index 0000000..cd7bb0a
--- /dev/null
+++ b/scripts/build-sidecar.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -ex
+if [ "${SKIP_SIDECAR_BUILD}" == "true" ]; then
+ echo "Skipping Sidecar Build because SKIP_SIDECAR_BUILD was set to 'true'"
+else
+ SCRIPT_DIR=$( dirname -- "$( readlink -f -- "$0"; )"; )
+ SIDECAR_REPO="${SIDECAR_REPO:-https://github.com/apache/cassandra-sidecar.git}"
+ SIDECAR_BRANCH="${SIDECAR_BRANCH:-trunk}"
+ SIDECAR_COMMIT="${SIDECAR_COMMIT:-0dd7a14f7e70dc8a40d20eba5c78dc4f6a580e17}"
+ SIDECAR_JAR_DIR="$(dirname "${SCRIPT_DIR}/")/dependencies"
+ SIDECAR_JAR_DIR=${CASSANDRA_DEP_DIR:-$SIDECAR_JAR_DIR}
+ SIDECAR_BUILD_DIR="${SIDECAR_JAR_DIR}/sidecar-build"
+ java_ver_output=`"${JAVA:-java}" -version 2>&1`
+ jvmver=`echo "$java_ver_output" | grep '[openjdk|java] version' | awk -F'"' 'NR==1 {print $2}' | cut -d\- -f1`
+ JVM_VERSION=${jvmver%_*}
+ echo $JVM_VERSION
+ if [ "${JVM_VERSION}" == "1.8.0" ]; then
+ SIDECAR_BUILD_VERSION="1.0.0-jdk8-analytics"
+ else
+ SIDECAR_BUILD_VERSION="1.0.0-analytics"
+ fi
+ if [[ "$CLEAN" == "true" ]]; then
+ echo "Clean up $SIDECAR_BUILD_DIR and $SIDECAR_JAR_DIR/org/apache/cassandra/(cassandra-)sidecar directories"
+ rm -rf "${SIDECAR_BUILD_DIR}"
+ rm -rf "${SIDECAR_JAR_DIR}/org/apache/cassandra/sidecar"
+ rm -rf "${SIDECAR_JAR_DIR}/org/apache/cassandra/cassandra-sidecar"
+ fi
+ mkdir -p "${SIDECAR_BUILD_DIR}"
+ cd "${SIDECAR_BUILD_DIR}"
+ echo "branch ${SIDECAR_BRANCH} sha ${SIDECAR_COMMIT}"
+ # check out the correct cassandra version:
+ if [ ! -d "${SIDECAR_BRANCH}" ] ; then
+ if [ -n "${SIDECAR_COMMIT}" ] ; then
+ mkdir -p "${SIDECAR_BRANCH}"
+ cd "${SIDECAR_BRANCH}"
+ git init
+ git remote add upstream "${SIDECAR_REPO}"
+ git fetch --depth=1 upstream "${SIDECAR_COMMIT}"
+ git reset --hard FETCH_HEAD
+ else
+ git clone --depth 1 --single-branch --branch "${SIDECAR_BRANCH}" "${SIDECAR_REPO}" "${SIDECAR_BRANCH}"
+ cd "${SIDECAR_BRANCH}"
+ fi
+ else
+ cd "${SIDECAR_BRANCH}"
+ if [ -z "${SIDECAR_COMMIT}" ] ; then
+ git pull
+ fi
+ fi
+ if [ -z "${SIDECAR_COMMIT}" ] ; then
+ git checkout "${SIDECAR_BRANCH}"
+ else
+ git checkout "${SIDECAR_COMMIT}"
+ fi
+ git clean -fd
+ ./gradlew -Pversion=${SIDECAR_BUILD_VERSION} -Dmaven.repo.local=${SIDECAR_JAR_DIR} publishToMavenLocal
+fi
diff --git a/scripts/functions.sh b/scripts/functions.sh
new file mode 100755
index 0000000..149afce
--- /dev/null
+++ b/scripts/functions.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+get_hostname() {
+ if [[ $# -eq 0 ]] ; then
+ echo 'An URI must be provided'
+ exit 1
+ fi
+
+ # removes the scheme portion of the URI (i.e. https://, http://)
+ hostname=$(echo "${1}" | sed -E 's/^\s*.*:\/\///g')
+ # removes the first encounter of a forward slash / until the end of the string
+ hostname=${hostname%%/*}
+ # removes the first encounter of a colon : until the end of the string i.e git@github.com:apache -> git@github.com
+ hostname=${hostname%%:*}
+ # removes the username portion of an URI i.e. git@github.com -> github.com
+ hostname=$(echo "${hostname}" | cut -d@ -f2-)
+
+ echo "${hostname}"
+}
+
+test_get_hostname() {
+ get_hostname "https://github.com/apache/cassandra-sidecar" # returns github.com
+ get_hostname "git@github.com:apache/cassandra-sidecar.git" # returns github.com
+ get_hostname # returns An URI must be provided message with exit code 1
+}
diff --git a/scripts/mvnw b/scripts/mvnw
new file mode 100755
index 0000000..8d937f4
--- /dev/null
+++ b/scripts/mvnw
@@ -0,0 +1,308 @@
+#!/bin/sh
+# ----------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ----------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------
+# Apache Maven Wrapper startup batch script, version 3.2.0
+#
+# Required ENV vars:
+# ------------------
+# JAVA_HOME - location of a JDK home dir
+#
+# Optional ENV vars
+# -----------------
+# MAVEN_OPTS - parameters passed to the Java VM when running Maven
+# e.g. to debug Maven itself, use
+# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
+# MAVEN_SKIP_RC - flag to disable loading of mavenrc files
+# ----------------------------------------------------------------------------
+
+if [ -z "$MAVEN_SKIP_RC" ] ; then
+
+ if [ -f /usr/local/etc/mavenrc ] ; then
+ . /usr/local/etc/mavenrc
+ fi
+
+ if [ -f /etc/mavenrc ] ; then
+ . /etc/mavenrc
+ fi
+
+ if [ -f "$HOME/.mavenrc" ] ; then
+ . "$HOME/.mavenrc"
+ fi
+
+fi
+
+# OS specific support. $var _must_ be set to either true or false.
+cygwin=false;
+darwin=false;
+mingw=false
+case "$(uname)" in
+ CYGWIN*) cygwin=true ;;
+ MINGW*) mingw=true;;
+ Darwin*) darwin=true
+ # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
+ # See https://developer.apple.com/library/mac/qa/qa1170/_index.html
+ if [ -z "$JAVA_HOME" ]; then
+ if [ -x "/usr/libexec/java_home" ]; then
+ JAVA_HOME="$(/usr/libexec/java_home)"; export JAVA_HOME
+ else
+ JAVA_HOME="/Library/Java/Home"; export JAVA_HOME
+ fi
+ fi
+ ;;
+esac
+
+if [ -z "$JAVA_HOME" ] ; then
+ if [ -r /etc/gentoo-release ] ; then
+ JAVA_HOME=$(java-config --jre-home)
+ fi
+fi
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched
+if $cygwin ; then
+ [ -n "$JAVA_HOME" ] &&
+ JAVA_HOME=$(cygpath --unix "$JAVA_HOME")
+ [ -n "$CLASSPATH" ] &&
+ CLASSPATH=$(cygpath --path --unix "$CLASSPATH")
+fi
+
+# For Mingw, ensure paths are in UNIX format before anything is touched
+if $mingw ; then
+ [ -n "$JAVA_HOME" ] && [ -d "$JAVA_HOME" ] &&
+ JAVA_HOME="$(cd "$JAVA_HOME" || (echo "cannot cd into $JAVA_HOME."; exit 1); pwd)"
+fi
+
+if [ -z "$JAVA_HOME" ]; then
+ javaExecutable="$(which javac)"
+ if [ -n "$javaExecutable" ] && ! [ "$(expr "\"$javaExecutable\"" : '\([^ ]*\)')" = "no" ]; then
+ # readlink(1) is not available as standard on Solaris 10.
+ readLink=$(which readlink)
+ if [ ! "$(expr "$readLink" : '\([^ ]*\)')" = "no" ]; then
+ if $darwin ; then
+ javaHome="$(dirname "\"$javaExecutable\"")"
+ javaExecutable="$(cd "\"$javaHome\"" && pwd -P)/javac"
+ else
+ javaExecutable="$(readlink -f "\"$javaExecutable\"")"
+ fi
+ javaHome="$(dirname "\"$javaExecutable\"")"
+ javaHome=$(expr "$javaHome" : '\(.*\)/bin')
+ JAVA_HOME="$javaHome"
+ export JAVA_HOME
+ fi
+ fi
+fi
+
+if [ -z "$JAVACMD" ] ; then
+ if [ -n "$JAVA_HOME" ] ; then
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+ # IBM's JDK on AIX uses strange locations for the executables
+ JAVACMD="$JAVA_HOME/jre/sh/java"
+ else
+ JAVACMD="$JAVA_HOME/bin/java"
+ fi
+ else
+ JAVACMD="$(\unset -f command 2>/dev/null; \command -v java)"
+ fi
+fi
+
+if [ ! -x "$JAVACMD" ] ; then
+ echo "Error: JAVA_HOME is not defined correctly." >&2
+ echo " We cannot execute $JAVACMD" >&2
+ exit 1
+fi
+
+if [ -z "$JAVA_HOME" ] ; then
+ echo "Warning: JAVA_HOME environment variable is not set."
+fi
+
+# traverses directory structure from process work directory to filesystem root
+# first directory with .mvn subdirectory is considered project base directory
+find_maven_basedir() {
+ if [ -z "$1" ]
+ then
+ echo "Path not specified to find_maven_basedir"
+ return 1
+ fi
+
+ basedir="$1"
+ wdir="$1"
+ while [ "$wdir" != '/' ] ; do
+ if [ -d "$wdir"/.mvn ] ; then
+ basedir=$wdir
+ break
+ fi
+ # workaround for JBEAP-8937 (on Solaris 10/Sparc)
+ if [ -d "${wdir}" ]; then
+ wdir=$(cd "$wdir/.." || exit 1; pwd)
+ fi
+ # end of workaround
+ done
+ printf '%s' "$(cd "$basedir" || exit 1; pwd)"
+}
+
+# concatenates all lines of a file
+concat_lines() {
+ if [ -f "$1" ]; then
+ # Remove \r in case we run on Windows within Git Bash
+ # and check out the repository with auto CRLF management
+ # enabled. Otherwise, we may read lines that are delimited with
+ # \r\n and produce $'-Xarg\r' rather than -Xarg due to word
+ # splitting rules.
+ tr -s '\r\n' ' ' < "$1"
+ fi
+}
+
+log() {
+ if [ "$MVNW_VERBOSE" = true ]; then
+ printf '%s\n' "$1"
+ fi
+}
+
+BASE_DIR=$(find_maven_basedir "$(dirname "$0")")
+if [ -z "$BASE_DIR" ]; then
+ exit 1;
+fi
+
+MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}; export MAVEN_PROJECTBASEDIR
+log "$MAVEN_PROJECTBASEDIR"
+
+##########################################################################################
+# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
+# This allows using the maven wrapper in projects that prohibit checking in binary data.
+##########################################################################################
+wrapperJarPath="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar"
+if [ -r "$wrapperJarPath" ]; then
+ log "Found $wrapperJarPath"
+else
+ log "Couldn't find $wrapperJarPath, downloading it ..."
+
+ if [ -n "$MVNW_REPOURL" ]; then
+ wrapperUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar"
+ else
+ wrapperUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar"
+ fi
+ while IFS="=" read -r key value; do
+ # Remove '\r' from value to allow usage on windows as IFS does not consider '\r' as a separator ( considers space, tab, new line ('\n'), and custom '=' )
+ safeValue=$(echo "$value" | tr -d '\r')
+ case "$key" in (wrapperUrl) wrapperUrl="$safeValue"; break ;;
+ esac
+ done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties"
+ log "Downloading from: $wrapperUrl"
+
+ if $cygwin; then
+ wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath")
+ fi
+
+ if command -v wget > /dev/null; then
+ log "Found wget ... using wget"
+ [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--quiet"
+ if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
+ wget $QUIET "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath"
+ else
+ wget $QUIET --http-user="$MVNW_USERNAME" --http-password="$MVNW_PASSWORD" "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath"
+ fi
+ elif command -v curl > /dev/null; then
+ log "Found curl ... using curl"
+ [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--silent"
+ if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
+ curl $QUIET -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath"
+ else
+ curl $QUIET --user "$MVNW_USERNAME:$MVNW_PASSWORD" -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath"
+ fi
+ else
+ log "Falling back to using Java to download"
+ javaSource="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.java"
+ javaClass="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.class"
+ # For Cygwin, switch paths to Windows format before running javac
+ if $cygwin; then
+ javaSource=$(cygpath --path --windows "$javaSource")
+ javaClass=$(cygpath --path --windows "$javaClass")
+ fi
+ if [ -e "$javaSource" ]; then
+ if [ ! -e "$javaClass" ]; then
+ log " - Compiling MavenWrapperDownloader.java ..."
+ ("$JAVA_HOME/bin/javac" "$javaSource")
+ fi
+ if [ -e "$javaClass" ]; then
+ log " - Running MavenWrapperDownloader.java ..."
+ ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$wrapperUrl" "$wrapperJarPath") || rm -f "$wrapperJarPath"
+ fi
+ fi
+ fi
+fi
+##########################################################################################
+# End of extension
+##########################################################################################
+
+# If specified, validate the SHA-256 sum of the Maven wrapper jar file
+wrapperSha256Sum=""
+while IFS="=" read -r key value; do
+ case "$key" in (wrapperSha256Sum) wrapperSha256Sum=$value; break ;;
+ esac
+done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties"
+if [ -n "$wrapperSha256Sum" ]; then
+ wrapperSha256Result=false
+ if command -v sha256sum > /dev/null; then
+ if echo "$wrapperSha256Sum $wrapperJarPath" | sha256sum -c > /dev/null 2>&1; then
+ wrapperSha256Result=true
+ fi
+ elif command -v shasum > /dev/null; then
+ if echo "$wrapperSha256Sum $wrapperJarPath" | shasum -a 256 -c > /dev/null 2>&1; then
+ wrapperSha256Result=true
+ fi
+ else
+ echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available."
+ echo "Please install either command, or disable validation by removing 'wrapperSha256Sum' from your maven-wrapper.properties."
+ exit 1
+ fi
+ if [ $wrapperSha256Result = false ]; then
+ echo "Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised." >&2
+ echo "Investigate or delete $wrapperJarPath to attempt a clean download." >&2
+ echo "If you updated your Maven version, you need to update the specified wrapperSha256Sum property." >&2
+ exit 1
+ fi
+fi
+
+MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin; then
+ [ -n "$JAVA_HOME" ] &&
+ JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME")
+ [ -n "$CLASSPATH" ] &&
+ CLASSPATH=$(cygpath --path --windows "$CLASSPATH")
+ [ -n "$MAVEN_PROJECTBASEDIR" ] &&
+ MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR")
+fi
+
+# Provide a "standardized" way to retrieve the CLI args that will
+# work with both Windows and non-Windows executions.
+MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $*"
+export MAVEN_CMD_LINE_ARGS
+
+WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
+
+# shellcheck disable=SC2086 # safe args
+exec "$JAVACMD" \
+ $MAVEN_OPTS \
+ $MAVEN_DEBUG_OPTS \
+ -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
+ "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
+ ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@"
diff --git a/scripts/relocate-dtest-dependencies.pom b/scripts/relocate-dtest-dependencies.pom
new file mode 100644
index 0000000..9c48bb7
--- /dev/null
+++ b/scripts/relocate-dtest-dependencies.pom
@@ -0,0 +1,192 @@
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.cassandra</groupId>
+ <artifactId>cassandra-dtest-shaded</artifactId>
+ <version>${revision}</version>
+ <packaging>jar</packaging>
+
+ <name>Cassandra in-jvm dtests shaded jar</name>
+
+ <properties>
+ <project.type>library</project.type>
+ <java.version>1.8</java.version>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+
+ <junit.version>4.12</junit.version>
+ <reflections.version>0.10.2</reflections.version>
+ <maven.compiler.source>${java.version}</maven.compiler.source>
+ <maven.compiler.target>${java.version}</maven.compiler.target>
+ <dtest-local.version>${dtest.version}</dtest-local.version>
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.cassandra</groupId>
+ <artifactId>cassandra-dtest-local</artifactId>
+ <version>${dtest-local.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- We need to include reflections in the final shaded jar explicitly
+ as 4.0 doesn't pull it in on its own due to a not-backported fix
+ in later dtest jars
+ -->
+ <dependency>
+ <groupId>org.reflections</groupId>
+ <artifactId>reflections</artifactId>
+ <version>${reflections.version}</version>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+
+ <plugins>
+ <!-- Skip main compilation as we only want to package existing dependencies -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>default-compile</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ <configuration>
+ <skipMain>true</skipMain>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- generate a shaded JAR -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.4.1</version>
+
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <createSourcesJar>false</createSourcesJar>
+
+ <artifactSet>
+ <includes>
+ <include>org.apache.cassandra:cassandra-dtest-local</include>
+ <include>org.reflections:reflections</include>
+ </includes>
+ </artifactSet>
+
+ <relocations>
+ <relocation>
+ <pattern>io.netty</pattern>
+ <shadedPattern>shaded.io.netty</shadedPattern>
+ </relocation>
+
+ <relocation>
+ <pattern>com.google</pattern>
+ <shadedPattern>shaded.com.google</shadedPattern>
+ <!-- ImmutableMaps and other items in com.google.common are shipped across the
+ network, so we need to keep them un-relocated -->
+ <excludes>
+ <exclude>com.google.common.**.*</exclude>
+ </excludes>
+ </relocation>
+
+ <relocation>
+ <pattern>com.github.benmanes.caffeine</pattern>
+ <shadedPattern>shaded.com.github.benmanes.caffeine</shadedPattern>
+ </relocation>
+
+ <relocation>
+ <pattern>com.datastax</pattern>
+ <shadedPattern>shaded.com.datastax</shadedPattern>
+ </relocation>
+
+ <relocation>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>shaded.com.fasterxml.jackson</shadedPattern>
+ </relocation>
+ </relocations>
+
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>**/Log4j2Plugins.dat</exclude>
+ </excludes>
+ </filter>
+
+ <filter>
+ <artifact>io.netty:netty-*</artifact>
+ <excludes>
+ <exclude>META-INF/maven/</exclude>
+ <exclude>META-INF/io.netty.versions.properties</exclude>
+ </excludes>
+ </filter>
+
+ <filter>
+ <artifact>com.google.guava:guava</artifact>
+ <excludes>
+ <exclude>META-INF/maven/</exclude>
+ </excludes>
+ </filter>
+
+ <filter>
+ <artifact>com.google.guava:failureaccess</artifact>
+ <excludes>
+ <exclude>META-INF/maven/</exclude>
+ </excludes>
+ </filter>
+
+ <filter>
+ <artifact>com.datastax.cassandra:cassandra-driver-core</artifact>
+ <excludes>
+ <exclude>META-INF/maven/</exclude>
+ </excludes>
+ </filter>
+
+ </filters>
+ <outputFile>${outputFilePath}</outputFile>
+ </configuration>
+ </execution>
+ </executions>
+
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/settings.gradle b/settings.gradle
index 704e124..7296546 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -23,3 +23,5 @@ include 'cassandra-bridge'
include 'cassandra-four-zero'
include 'cassandra-analytics-core'
include 'cassandra-analytics-core-example'
+include 'cassandra-analytics-integration-framework'
+include 'cassandra-analytics-integration-tests'
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org