You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2020/01/15 01:31:02 UTC
[incubator-pinot] branch master updated (5919af1 -> 29c6b39)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from 5919af1 QueryServer need not be a runnable (#4980)
add 29c6b39 Adding pinot-batch-ingestion-spark (#4975)
No new revisions were added by this update.
Summary of changes:
docs/batch_data_ingestion.rst | 30 +++-
pinot-distribution/pinot-assembly.xml | 3 +-
pinot-distribution/pom.xml | 36 +++--
.../pinot-batch-ingestion-common}/pom.xml | 20 ++-
.../batch/common}/SegmentGenerationTaskRunner.java | 12 +-
.../ingestion/batch/common/SegmentPushUtils.java} | 141 ++++++++---------
.../pinot-batch-ingestion-spark/pom.xml | 169 +++++++++++++++++++++
.../spark/SparkSegmentGenerationJobRunner.java} | 134 +++++++++++-----
.../batch/spark/SparkSegmentTarPushJobRunner.java | 123 +++++++++++++++
.../batch/spark/SparkSegmentUriPushJobRunner.java | 128 ++++++++++++++++
.../segmentCreationAndTarPushJobSpec.yaml | 8 +-
.../segmentCreationAndUriPushJobSpec.yaml | 8 +-
.../src/main/resources/segmentCreationJobSpec.yaml | 8 +-
.../src/main/resources/segmentTarPushJobSpec.yaml | 8 +-
.../src/main/resources/segmentUriPushJobSpec.yaml | 8 +-
.../pinot-batch-ingestion-standalone/pom.xml | 51 +------
.../standalone/SegmentGenerationJobRunner.java | 13 +-
.../batch/standalone/SegmentTarPushJobRunner.java | 76 +--------
.../batch/standalone/SegmentUriPushJobRunner.java | 67 +-------
pinot-plugins/pinot-batch-ingestion/pom.xml | 6 +
.../v0_deprecated/pinot-hadoop/pom.xml | 18 ---
.../v0_deprecated/pinot-spark/pom.xml | 18 ---
pinot-plugins/pom.xml | 21 ++-
.../org/apache/pinot/spi/filesystem/PinotFS.java | 3 +-
.../batch}/IngestionJobLauncher.java | 8 +-
.../batch}/runner/IngestionJobRunner.java | 4 +-
.../batch}/spec/Constants.java | 2 +-
.../batch}/spec/ExecutionFrameworkSpec.java | 21 ++-
.../batch}/spec/PinotClusterSpec.java | 7 +-
.../batch}/spec/PinotFSSpec.java | 5 +-
.../batch}/spec/PushJobSpec.java | 20 ++-
.../batch}/spec/RecordReaderSpec.java | 5 +-
.../batch}/spec/SegmentGenerationJobSpec.java | 6 +-
.../batch}/spec/SegmentGenerationTaskSpec.java | 5 +-
.../batch}/spec/SegmentNameGeneratorSpec.java | 5 +-
.../batch}/spec/TableSpec.java | 7 +-
pinot-tools/pom.xml | 2 +-
.../command/LaunchDataIngestionJobCommand.java | 2 +-
.../tools/admin/command/QuickstartRunner.java | 2 +-
...tionJobSpec.yaml => sparkIngestionJobSpec.yaml} | 28 +++-
...tionJobSpec.yaml => sparkIngestionJobSpec.yaml} | 14 +-
41 files changed, 841 insertions(+), 411 deletions(-)
copy pinot-plugins/{pinot-stream-ingestion/pinot-kafka-base => pinot-batch-ingestion/pinot-batch-ingestion-common}/pom.xml (73%)
rename pinot-plugins/pinot-batch-ingestion/{pinot-batch-ingestion-standalone/src/main/java/org/apache/pinot/plugin/ingestion/batch/standalone => pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common}/SegmentGenerationTaskRunner.java (94%)
copy pinot-plugins/pinot-batch-ingestion/{pinot-batch-ingestion-standalone/src/main/java/org/apache/pinot/plugin/ingestion/batch/standalone/SegmentTarPushJobRunner.java => pinot-batch-ingestion-common/src/main/java/org/apache/pinot/plugin/ingestion/batch/common/SegmentPushUtils.java} (52%)
create mode 100644 pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/pom.xml
copy pinot-plugins/pinot-batch-ingestion/{pinot-batch-ingestion-standalone/src/main/java/org/apache/pinot/plugin/ingestion/batch/standalone/SegmentGenerationJobRunner.java => pinot-batch-ingestion-spark/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/SparkSegmentGenerationJobRunner.java} (71%)
create mode 100644 pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/SparkSegmentTarPushJobRunner.java
create mode 100644 pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark/src/main/java/org/apache/pinot/plugin/ingestion/batch/spark/SparkSegmentUriPushJobRunner.java
copy pinot-plugins/pinot-batch-ingestion/{pinot-batch-ingestion-standalone => pinot-batch-ingestion-spark}/src/main/resources/segmentCreationAndTarPushJobSpec.yaml (89%)
copy pinot-plugins/pinot-batch-ingestion/{pinot-batch-ingestion-standalone => pinot-batch-ingestion-spark}/src/main/resources/segmentCreationAndUriPushJobSpec.yaml (89%)
copy pinot-plugins/pinot-batch-ingestion/{pinot-batch-ingestion-standalone => pinot-batch-ingestion-spark}/src/main/resources/segmentCreationJobSpec.yaml (90%)
copy pinot-plugins/pinot-batch-ingestion/{pinot-batch-ingestion-standalone => pinot-batch-ingestion-spark}/src/main/resources/segmentTarPushJobSpec.yaml (88%)
copy pinot-plugins/pinot-batch-ingestion/{pinot-batch-ingestion-standalone => pinot-batch-ingestion-spark}/src/main/resources/segmentUriPushJobSpec.yaml (89%)
rename pinot-spi/src/main/java/org/apache/pinot/spi/{batch/ingestion => ingestion/batch}/IngestionJobLauncher.java (94%)
rename pinot-spi/src/main/java/org/apache/pinot/spi/{batch/ingestion => ingestion/batch}/runner/IngestionJobRunner.java (91%)
rename pinot-spi/src/main/java/org/apache/pinot/spi/{batch/ingestion => ingestion/batch}/spec/Constants.java (95%)
rename pinot-spi/src/main/java/org/apache/pinot/spi/{batch/ingestion => ingestion/batch}/spec/ExecutionFrameworkSpec.java (84%)
rename pinot-spi/src/main/java/org/apache/pinot/spi/{batch/ingestion => ingestion/batch}/spec/PinotClusterSpec.java (90%)
rename pinot-spi/src/main/java/org/apache/pinot/spi/{batch/ingestion => ingestion/batch}/spec/PinotFSSpec.java (94%)
rename pinot-spi/src/main/java/org/apache/pinot/spi/{batch/ingestion => ingestion/batch}/spec/PushJobSpec.java (87%)
rename pinot-spi/src/main/java/org/apache/pinot/spi/{batch/ingestion => ingestion/batch}/spec/RecordReaderSpec.java (96%)
rename pinot-spi/src/main/java/org/apache/pinot/spi/{batch/ingestion => ingestion/batch}/spec/SegmentGenerationJobSpec.java (97%)
rename pinot-spi/src/main/java/org/apache/pinot/spi/{batch/ingestion => ingestion/batch}/spec/SegmentGenerationTaskSpec.java (95%)
rename pinot-spi/src/main/java/org/apache/pinot/spi/{batch/ingestion => ingestion/batch}/spec/SegmentNameGeneratorSpec.java (90%)
rename pinot-spi/src/main/java/org/apache/pinot/spi/{batch/ingestion => ingestion/batch}/spec/TableSpec.java (93%)
copy pinot-tools/src/main/resources/examples/batch/airlineStats/{ingestionJobSpec.yaml => sparkIngestionJobSpec.yaml} (86%)
copy pinot-tools/src/main/resources/examples/batch/baseballStats/{ingestionJobSpec.yaml => sparkIngestionJobSpec.yaml} (96%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org