You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ma...@apache.org on 2022/06/02 20:43:08 UTC
[solr-sandbox] branch crossdc-wip updated: Simplify build artifacts. (#20)
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch crossdc-wip
in repository https://gitbox.apache.org/repos/asf/solr-sandbox.git
The following commit(s) were added to refs/heads/crossdc-wip by this push:
new dd92247 Simplify build artifacts. (#20)
dd92247 is described below
commit dd922476c616a31f549ccbdadc02de79ca4f17b9
Author: Mark Robert Miller <ma...@apache.org>
AuthorDate: Thu Jun 2 15:43:05 2022 -0500
Simplify build artifacts. (#20)
* Revert "Experiment with Producer in crossdc package. (#19)"
This reverts commit 9007aa6b8d51384ab5f146ebdce013633bae4660.
* Simplify build artifacts and allow for future shading if necessary with commons and producer only creating a single uber jar artifact.
---
cluster.sh | 5 ++-
crossdc-commons/build.gradle | 25 ++++++++++++-
.../apache/solr/update/processor/UpdateHelper.java | 27 --------------
crossdc-consumer/build.gradle | 7 +---
crossdc-producer/build.gradle | 43 +++++++++++++---------
.../processor/KafkaRequestMirroringHandler.java | 3 +-
.../update/processor/MirroringException.java | 2 +-
.../update/processor/MirroringUpdateProcessor.java | 15 +++-----
.../MirroringUpdateRequestProcessorFactory.java | 5 +--
.../update/processor/RequestMirroringHandler.java | 2 +-
.../configs/cloud-minimal/conf/solrconfig.xml | 2 +-
crossdc-producer/src/test/resources/log4j2.xml | 8 ++--
12 files changed, 69 insertions(+), 75 deletions(-)
diff --git a/cluster.sh b/cluster.sh
index e7439df..18e0bb8 100644
--- a/cluster.sh
+++ b/cluster.sh
@@ -41,6 +41,8 @@ bin/zookeeper-server-start.sh config/zookeeper.properties > ../kafka_zk.log &
bin/kafka-server-start.sh config/server.properties > ../kafka_server.log &
+# The following commented out section is just for helpful reference
+
# for kafka 2.x zk port of 2181, for 3.x broker of 9093
# bin/kafka-topics.sh --create --topic my-kafka-topic --bootstrap-server localhost:9093 --partitions 3 --replication-factor 2
@@ -56,9 +58,8 @@ bin/kafka-server-start.sh config/server.properties > ../kafka_server.log &
# need to go to lib folder - I can't believe there is no shared lib folder by default - crazy
mkdir "${solr}/server/solr/lib"
-cp ../crossdc-commons/build/libs/crossdc-commons-*.jar "${solr}"/server/solr/lib
+
cp ../crossdc-producer/build/libs/crossdc-producer-*.jar "${solr}"/server/solr/lib
-cp ../crossdc-producer/build/libs/kafka-clients-*.jar "${solr}"/server/solr/lib
(
cd "${solr}" || exit
diff --git a/crossdc-commons/build.gradle b/crossdc-commons/build.gradle
index 9e97f6c..26e1893 100644
--- a/crossdc-commons/build.gradle
+++ b/crossdc-commons/build.gradle
@@ -16,6 +16,7 @@
*/
plugins {
id 'java'
+ id 'com.github.johnrengelman.shadow' version '7.1.2'
}
description = 'Cross-DC Commons package'
@@ -27,9 +28,16 @@ repositories {
jcenter()
}
+configurations {
+ provided
+}
+
+sourceSets {
+ main { compileClasspath += configurations.provided }
+}
+
dependencies {
- implementation 'org.apache.solr:solr-solrj:8.11.1'
- implementation 'org.apache.solr:solr-core:8.11.1'
+ provided 'org.apache.solr:solr-solrj:8.11.1'
implementation 'org.apache.kafka:kafka-clients:2.8.1'
implementation 'com.google.guava:guava:14.0'
}
@@ -38,6 +46,19 @@ subprojects {
group "org.apache.solr"
}
+jar.enabled = false
+
+shadowJar {
+ archiveBaseName.set('crossdc-commons')
+ configurations = [project.configurations.compileClasspath]
+}
+
+jar.dependsOn(shadowJar)
+
+artifacts {
+ shadowJar;
+}
+
test {
jvmArgs '-Djava.security.egd=file:/dev/./urandom'
}
diff --git a/crossdc-commons/src/main/java/org/apache/solr/update/processor/UpdateHelper.java b/crossdc-commons/src/main/java/org/apache/solr/update/processor/UpdateHelper.java
deleted file mode 100644
index 4466dd3..0000000
--- a/crossdc-commons/src/main/java/org/apache/solr/update/processor/UpdateHelper.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.update.processor;
-
-public class UpdateHelper {
- public static boolean isLeader(DistributedUpdateProcessor distProc) {
- return distProc.isLeader();
- }
-
- public static UpdateRequestProcessor next(UpdateRequestProcessor proc) {
- return proc.next;
- }
-}
diff --git a/crossdc-consumer/build.gradle b/crossdc-consumer/build.gradle
index 79dc7c8..e49e8e0 100644
--- a/crossdc-consumer/build.gradle
+++ b/crossdc-consumer/build.gradle
@@ -33,16 +33,12 @@ application {
dependencies {
implementation group: 'org.apache.solr', name: 'solr-solrj', version: '8.11.1'
- implementation (project(':crossdc-commons')) {
- exclude (group: 'org.apache.solr', module: 'solr-core')
- }
+ implementation project(path: ':crossdc-commons', configuration: 'shadow')
implementation 'org.slf4j:slf4j-api:1.7.36'
implementation 'org.eclipse.jetty:jetty-http:9.4.41.v20210516'
implementation 'org.eclipse.jetty:jetty-server:9.4.41.v20210516'
implementation 'org.eclipse.jetty:jetty-servlet:9.4.41.v20210516'
- implementation 'org.apache.kafka:kafka-clients:2.8.1'
- implementation group: 'com.google.guava', name: 'guava', version: '14.0'
implementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.17.2'
runtimeOnly ('com.google.protobuf:protobuf-java-util:3.19.2')
testImplementation 'org.hamcrest:hamcrest:2.2'
@@ -58,7 +54,6 @@ dependencies {
testImplementation 'org.apache.kafka:kafka_2.13:2.8.1'
testImplementation 'org.apache.kafka:kafka-streams:2.8.1'
testImplementation 'org.apache.kafka:kafka_2.13:2.8.1:test'
- testImplementation 'org.apache.kafka:kafka-clients:2.8.1:test'
testImplementation 'org.apache.kafka:kafka-streams:2.8.1:test'
}
diff --git a/crossdc-producer/build.gradle b/crossdc-producer/build.gradle
index 1447b7a..b636c88 100644
--- a/crossdc-producer/build.gradle
+++ b/crossdc-producer/build.gradle
@@ -16,6 +16,7 @@
*/
plugins {
id 'java'
+ id 'com.github.johnrengelman.shadow' version '7.1.2'
}
description = 'Cross-DC Producer package'
@@ -26,18 +27,21 @@ repositories {
mavenCentral()
}
+configurations {
+ provided
+}
+
+sourceSets {
+ main { compileClasspath += configurations.provided }
+}
+
dependencies {
implementation project(':crossdc-consumer')
- implementation project(':crossdc-commons')
- implementation group: 'org.apache.solr', name: 'solr-solrj', version: '8.11.1'
- implementation group: 'org.apache.solr', name: 'solr-core', version: '8.11.1'
- implementation 'org.slf4j:slf4j-api'
- implementation 'org.eclipse.jetty:jetty-http:9.4.41.v20210516'
- implementation 'org.eclipse.jetty:jetty-server:9.4.41.v20210516'
- implementation 'org.eclipse.jetty:jetty-servlet:9.4.41.v20210516'
- implementation 'org.apache.kafka:kafka-clients:2.8.0'
- implementation group: 'com.google.guava', name: 'guava', version: '14.0'
- runtimeOnly ('com.google.protobuf:protobuf-java-util:3.19.2')
+ implementation project(path: ':crossdc-commons', configuration: 'shadow')
+
+ provided group: 'org.apache.solr', name: 'solr-core', version: '8.11.1'
+
+ testImplementation 'org.slf4j:slf4j-api'
testImplementation 'org.hamcrest:hamcrest:2.2'
testImplementation 'junit:junit:4.13.2'
testImplementation('org.mockito:mockito-core:4.3.1', {
@@ -45,26 +49,31 @@ dependencies {
})
testImplementation group: 'org.apache.solr', name: 'solr-core', version: '8.11.1'
testImplementation group: 'org.apache.solr', name: 'solr-test-framework', version: '8.11.1'
+
testImplementation 'org.apache.kafka:kafka_2.13:2.8.1'
testImplementation 'org.apache.kafka:kafka-streams:2.8.1'
-
testImplementation 'org.apache.kafka:kafka_2.13:2.8.1:test'
- testImplementation 'org.apache.kafka:kafka-clients:2.8.1:test'
testImplementation 'org.apache.kafka:kafka-streams:2.8.1:test'
+ testImplementation 'org.apache.kafka:kafka-clients:2.8.1:test'
}
subprojects {
group "org.apache.solr"
}
-task copyToLib(type: Copy) {
- into "$buildDir/libs"
- include "kafka-clients-*.jar"
- from configurations.runtimeClasspath
+jar.enabled = false
+
+shadowJar {
+ archiveBaseName.set('crossdc-producer')
+ configurations = [project.configurations.compileClasspath]
}
-jar.dependsOn(copyToLib)
+jar.dependsOn(shadowJar)
+
+artifacts {
+ shadowJar;
+}
test {
jvmArgs '-Djava.security.egd=file:/dev/./urandom'
diff --git a/crossdc-producer/src/main/java/org/apache/solr/crossdc/update/processor/KafkaRequestMirroringHandler.java b/crossdc-producer/src/main/java/org/apache/solr/update/processor/KafkaRequestMirroringHandler.java
similarity index 95%
rename from crossdc-producer/src/main/java/org/apache/solr/crossdc/update/processor/KafkaRequestMirroringHandler.java
rename to crossdc-producer/src/main/java/org/apache/solr/update/processor/KafkaRequestMirroringHandler.java
index 4913fa3..34bc29b 100644
--- a/crossdc-producer/src/main/java/org/apache/solr/crossdc/update/processor/KafkaRequestMirroringHandler.java
+++ b/crossdc-producer/src/main/java/org/apache/solr/update/processor/KafkaRequestMirroringHandler.java
@@ -14,11 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.crossdc.update.processor;
+package org.apache.solr.update.processor;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.crossdc.common.KafkaMirroringSink;
import org.apache.solr.crossdc.common.MirroringException;
+import org.apache.solr.crossdc.common.KafkaCrossDcConf;
import org.apache.solr.crossdc.common.MirroredSolrRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/crossdc-producer/src/main/java/org/apache/solr/crossdc/update/processor/MirroringException.java b/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringException.java
similarity index 96%
rename from crossdc-producer/src/main/java/org/apache/solr/crossdc/update/processor/MirroringException.java
rename to crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringException.java
index d5e7c68..6f00df9 100644
--- a/crossdc-producer/src/main/java/org/apache/solr/crossdc/update/processor/MirroringException.java
+++ b/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringException.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.crossdc.update.processor;
+package org.apache.solr.update.processor;
/**
* Wrapper class for Mirroring exceptions.
diff --git a/crossdc-producer/src/main/java/org/apache/solr/crossdc/update/processor/MirroringUpdateProcessor.java b/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateProcessor.java
similarity index 91%
rename from crossdc-producer/src/main/java/org/apache/solr/crossdc/update/processor/MirroringUpdateProcessor.java
rename to crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateProcessor.java
index 06a3344..e2e09f2 100644
--- a/crossdc-producer/src/main/java/org/apache/solr/crossdc/update/processor/MirroringUpdateProcessor.java
+++ b/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateProcessor.java
@@ -1,4 +1,4 @@
-package org.apache.solr.crossdc.update.processor;
+package org.apache.solr.update.processor;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.SolrException;
@@ -10,9 +10,6 @@ import org.apache.solr.update.AddUpdateCommand;
import org.apache.solr.update.CommitUpdateCommand;
import org.apache.solr.update.DeleteUpdateCommand;
import org.apache.solr.update.RollbackUpdateCommand;
-import org.apache.solr.update.processor.DistributedUpdateProcessor;
-import org.apache.solr.update.processor.UpdateHelper;
-import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,7 +57,7 @@ public class MirroringUpdateProcessor extends UpdateRequestProcessor {
this.requestMirroringHandler = requestMirroringHandler;
// Find the downstream distributed update processor
- for (UpdateRequestProcessor proc = next; proc != null; proc = UpdateHelper.next(proc)) {
+ for (UpdateRequestProcessor proc = next; proc != null; proc = proc.next) {
if (proc instanceof DistributedUpdateProcessor) {
distProc = (DistributedUpdateProcessor) proc;
break;
@@ -86,11 +83,11 @@ public class MirroringUpdateProcessor extends UpdateRequestProcessor {
@Override public void processAdd(final AddUpdateCommand cmd) throws IOException {
if (log.isDebugEnabled())
- log.debug("processAdd isLeader={} cmd={}", UpdateHelper.isLeader(distProc), cmd);
+ log.debug("processAdd isLeader={} cmd={}", distProc.isLeader(), cmd);
super.processAdd(cmd); // let this throw to prevent mirroring invalid reqs
// submit only from the leader shards so we mirror each doc once
- if (doMirroring && UpdateHelper.isLeader(distProc)) {
+ if (doMirroring && distProc.isLeader()) {
SolrInputDocument doc = cmd.getSolrInputDocument().deepCopy();
doc.removeField(CommonParams.VERSION_FIELD); // strip internal doc version
createAndOrGetMirrorRequest().add(doc, cmd.commitWithin, cmd.overwrite);
@@ -99,13 +96,13 @@ public class MirroringUpdateProcessor extends UpdateRequestProcessor {
@Override public void processDelete(final DeleteUpdateCommand cmd) throws IOException {
if (log.isDebugEnabled())
- log.debug("processDelete doMirroring={} isLeader={} cmd={}", doMirroring, UpdateHelper.isLeader(distProc), cmd);
+ log.debug("processDelete doMirroring={} isLeader={} cmd={}", doMirroring, distProc.isLeader(), cmd);
super.processDelete(cmd); // let this throw to prevent mirroring invalid requests
if (doMirroring) {
if (cmd.isDeleteById()) {
// deleteById requests runs once per leader, so we just submit the request from the leader shard
- if (UpdateHelper.isLeader(distProc)) {
+ if (distProc.isLeader()) {
createAndOrGetMirrorRequest().deleteById(cmd.getId()); // strip versions from deletes
}
} else {
diff --git a/crossdc-producer/src/main/java/org/apache/solr/crossdc/update/processor/MirroringUpdateRequestProcessorFactory.java b/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateRequestProcessorFactory.java
similarity index 96%
rename from crossdc-producer/src/main/java/org/apache/solr/crossdc/update/processor/MirroringUpdateRequestProcessorFactory.java
rename to crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateRequestProcessorFactory.java
index 43b836d..2d91531 100644
--- a/crossdc-producer/src/main/java/org/apache/solr/crossdc/update/processor/MirroringUpdateRequestProcessorFactory.java
+++ b/crossdc-producer/src/main/java/org/apache/solr/update/processor/MirroringUpdateRequestProcessorFactory.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.crossdc.update.processor;
+package org.apache.solr.update.processor;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
@@ -27,9 +27,6 @@ import org.apache.solr.crossdc.common.KafkaCrossDcConf;
import org.apache.solr.crossdc.common.KafkaMirroringSink;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.update.processor.DocBasedVersionConstraintsProcessorFactory;
-import org.apache.solr.update.processor.UpdateRequestProcessor;
-import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
import org.apache.solr.util.plugin.SolrCoreAware;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/crossdc-producer/src/main/java/org/apache/solr/crossdc/update/processor/RequestMirroringHandler.java b/crossdc-producer/src/main/java/org/apache/solr/update/processor/RequestMirroringHandler.java
similarity index 95%
rename from crossdc-producer/src/main/java/org/apache/solr/crossdc/update/processor/RequestMirroringHandler.java
rename to crossdc-producer/src/main/java/org/apache/solr/update/processor/RequestMirroringHandler.java
index ff55b6d..16ca862 100644
--- a/crossdc-producer/src/main/java/org/apache/solr/crossdc/update/processor/RequestMirroringHandler.java
+++ b/crossdc-producer/src/main/java/org/apache/solr/update/processor/RequestMirroringHandler.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.crossdc.update.processor;
+package org.apache.solr.update.processor;
import org.apache.solr.client.solrj.request.UpdateRequest;
diff --git a/crossdc-producer/src/test/resources/configs/cloud-minimal/conf/solrconfig.xml b/crossdc-producer/src/test/resources/configs/cloud-minimal/conf/solrconfig.xml
index 9090598..6e057ab 100644
--- a/crossdc-producer/src/test/resources/configs/cloud-minimal/conf/solrconfig.xml
+++ b/crossdc-producer/src/test/resources/configs/cloud-minimal/conf/solrconfig.xml
@@ -109,7 +109,7 @@
</query>
<updateRequestProcessorChain name="mirrorUpdateChain" default="true">
- <processor class="org.apache.solr.crossdc.update.processor.MirroringUpdateRequestProcessorFactory">
+ <processor class="org.apache.solr.update.processor.MirroringUpdateRequestProcessorFactory">
</processor>
<processor class="solr.LogUpdateProcessorFactory" />
diff --git a/crossdc-producer/src/test/resources/log4j2.xml b/crossdc-producer/src/test/resources/log4j2.xml
index 5f7ec24..3fd55ed 100644
--- a/crossdc-producer/src/test/resources/log4j2.xml
+++ b/crossdc-producer/src/test/resources/log4j2.xml
@@ -29,8 +29,8 @@
<RollingRandomAccessFile
name="MainLogFile"
- fileName="${sys:log.dir:-logs}/${sys:log.name:-crossdc}.log"
- filePattern="${sys:log.dir:-logs}/${sys:log.name:-crossdc}.log.%i">
+ fileName="${sys:log.dir:-build/logs}/${sys:log.name:-crossdc}.log"
+ filePattern="${sys:log.dir:-build/logs}/${sys:log.name:-crossdc}.log.%i">
<PatternLayout>
<Pattern>
%maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}] %c{1.}
@@ -60,8 +60,8 @@
<Logger name="org.eclipse.jetty" level="INFO"/>
<Logger name="org.apache.solr.crossdc.consumer.KafkaCrossDcConsumer" level="TRACE"/>
- <Logger name="org.apache.solr.crossdc.update.processor.MirroringUpdateProcessor" level="TRACE"/>
- <Logger name="org.apache.solr.crossdc.update.processor.KafkaRequestMirroringHandler" level="TRACE"/>
+ <Logger name="org.apache.solr.update.processor.MirroringUpdateProcessor" level="TRACE"/>
+ <Logger name="org.apache.solr.update.processor.KafkaRequestMirroringHandler" level="TRACE"/>
<Logger name="org.apache.solr.crossdc.messageprocessor.SolrMessageProcessor" level="TRACE"/>
<Logger name="org.apache.solr.crossdc.common.KafkaMirroringSink" level="TRACE"/>