You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2025/08/01 19:30:58 UTC
(pinot) branch master updated: Fix linter violation in SparkSegmentTarPushJobRunner and also apply fix for spark2 (#16488)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new e59aa47c352 Fix linter violation in SparkSegmentTarPushJobRunner and also apply fix for spark2 (#16488)
e59aa47c352 is described below
commit e59aa47c3525cdde3e0d8db3bb31fa874ab30cfb
Author: Xiaotian (Jackie) Jiang <17...@users.noreply.github.com>
AuthorDate: Fri Aug 1 13:30:53 2025 -0600
Fix linter violation in SparkSegmentTarPushJobRunner and also apply fix for spark2 (#16488)
---
.../ingestion/batch/spark/SparkSegmentTarPushJobRunner.java | 10 ++++------
.../ingestion/batch/spark3/SparkSegmentTarPushJobRunner.java | 11 ++++++-----
2 files changed, 10 insertions(+), 11 deletions(-)
diff --git a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/SparkSegmentTarPushJobRunner.java b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/SparkSegmentTarPushJobRunner.java
index babe30e7690..499ec2daba6 100644
--- a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/SparkSegmentTarPushJobRunner.java
+++ b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/SparkSegmentTarPushJobRunner.java
@@ -36,9 +36,7 @@ import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.VoidFunction;
-
public class SparkSegmentTarPushJobRunner extends BaseSparkSegmentTarPushJobRunner {
- private SegmentGenerationJobSpec _spec;
public SparkSegmentTarPushJobRunner() {
super();
@@ -48,8 +46,8 @@ public class SparkSegmentTarPushJobRunner extends BaseSparkSegmentTarPushJobRunn
super(spec);
}
- public void parallelizeTarPushJob(List<PinotFSSpec> pinotFSSpecs,
- List<String> segmentUris, int pushParallelism, URI outputDirURI) {
+ public void parallelizeTarPushJob(List<PinotFSSpec> pinotFSSpecs, List<String> segmentUris, int pushParallelism,
+ URI outputDirURI) {
JavaSparkContext sparkContext = JavaSparkContext.fromSparkContext(SparkContext.getOrCreate());
JavaRDD<String> pathRDD = sparkContext.parallelize(segmentUris, pushParallelism);
URI finalOutputDirURI = outputDirURI;
@@ -61,8 +59,8 @@ public class SparkSegmentTarPushJobRunner extends BaseSparkSegmentTarPushJobRunn
throws Exception {
PluginManager.get().init();
for (PinotFSSpec pinotFSSpec : pinotFSSpecs) {
- PinotFSFactory
- .register(pinotFSSpec.getScheme(), pinotFSSpec.getClassName(), new PinotConfiguration(pinotFSSpec));
+ PinotFSFactory.register(pinotFSSpec.getScheme(), pinotFSSpec.getClassName(),
+ new PinotConfiguration(pinotFSSpec));
}
try {
SegmentPushUtils.pushSegments(_spec, PinotFSFactory.create(finalOutputDirURI.getScheme()),
diff --git a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentTarPushJobRunner.java b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentTarPushJobRunner.java
index 4abfc449f39..ca27dd2b983 100644
--- a/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentTarPushJobRunner.java
+++ b/pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-3/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark3/SparkSegmentTarPushJobRunner.java
@@ -35,8 +35,9 @@ import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.VoidFunction;
+
public class SparkSegmentTarPushJobRunner extends BaseSparkSegmentTarPushJobRunner {
-
+
public SparkSegmentTarPushJobRunner() {
super();
}
@@ -45,8 +46,8 @@ public class SparkSegmentTarPushJobRunner extends BaseSparkSegmentTarPushJobRunn
super(spec);
}
- public void parallelizeTarPushJob(List<PinotFSSpec> pinotFSSpecs,
- List<String> segmentUris, int pushParallelism, URI outputDirURI) {
+ public void parallelizeTarPushJob(List<PinotFSSpec> pinotFSSpecs, List<String> segmentUris, int pushParallelism,
+ URI outputDirURI) {
JavaSparkContext sparkContext = JavaSparkContext.fromSparkContext(SparkContext.getOrCreate());
JavaRDD<String> pathRDD = sparkContext.parallelize(segmentUris, pushParallelism);
URI finalOutputDirURI = outputDirURI;
@@ -58,8 +59,8 @@ public class SparkSegmentTarPushJobRunner extends BaseSparkSegmentTarPushJobRunn
throws Exception {
PluginManager.get().init();
for (PinotFSSpec pinotFSSpec : pinotFSSpecs) {
- PinotFSFactory
- .register(pinotFSSpec.getScheme(), pinotFSSpec.getClassName(), new PinotConfiguration(pinotFSSpec));
+ PinotFSFactory.register(pinotFSSpec.getScheme(), pinotFSSpec.getClassName(),
+ new PinotConfiguration(pinotFSSpec));
}
try {
SegmentPushUtils.pushSegments(_spec, PinotFSFactory.create(finalOutputDirURI.getScheme()),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org