You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2019/11/30 20:00:57 UTC
[incubator-pinot] branch master updated (63ceb0c -> 3e7cbfa)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from 63ceb0c Adding pinot-spi module and moving record reader interface (#4869)
add 3e7cbfa Pinot Ingestion Jobs Refactor with Pinot Spark Support (#4800)
No new revisions were added by this update.
Summary of changes:
docs/pinot_hadoop.rst | 104 ++++++++++++
pinot-hadoop/README.md | 78 ---------
.../pinot-hadoop}/README.md | 6 +-
.../pinot-hadoop}/pom.xml | 9 +-
.../pinot/hadoop/PinotHadoopJobLauncher.java | 30 ++--
.../pinot/hadoop/io/CombineAvroKeyInputFormat.java | 0
.../org/apache/pinot/hadoop/io/FileHandler.java | 4 +-
.../pinot/hadoop/io/JsonPinotOutputFormat.java | 0
.../apache/pinot/hadoop/io/PinotOutputFormat.java | 0
.../org/apache/pinot/hadoop/io/PinotRecord.java | 0
.../pinot/hadoop/io/PinotRecordSerialization.java | 0
.../apache/pinot/hadoop/io/PinotRecordWriter.java | 2 +-
.../pinot/hadoop/job/HadoopSegmentCreationJob.java | 108 ++-----------
.../hadoop/job/HadoopSegmentPreprocessingJob.java | 129 ++++++---------
.../pinot/hadoop/job/InternalConfigConstants.java | 0
.../hadoop/job/mappers/SegmentCreationMapper.java | 2 +-
.../job/mappers/SegmentPreprocessingMapper.java | 8 +-
.../job/partitioners/GenericPartitioner.java | 8 +-
.../job/partitioners/PartitionFunctionFactory.java | 5 +-
.../job/reducers/SegmentPreprocessingReducer.java | 1 +
.../utils/PinotHadoopJobPreparationHelper.java | 25 +--
.../hadoop/io/DeleteExtraPushedSegmentsTest.java | 4 +-
.../pinot/hadoop/io/PinotOutputFormatTest.java | 0
.../hadoop/job/mappers/SegmentCreationTest.java | 0
.../pinot-ingestion-common}/pom.xml | 78 +++++----
.../pinot/ingestion/common}/ControllerRestApi.java | 2 +-
.../common}/DefaultControllerRestApi.java | 63 +++++---
.../ingestion/common}/JobConfigConstants.java | 16 +-
.../ingestion/common/PinotIngestionJobType.java | 14 +-
.../pinot/ingestion/jobs}/BaseSegmentJob.java | 55 ++++++-
.../pinot/ingestion/jobs/SegmentCreationJob.java | 169 +++++++++++++++++++
.../ingestion/jobs/SegmentPreprocessingJob.java | 115 +++++++++++++
.../pinot/ingestion/jobs}/SegmentTarPushJob.java | 30 ++--
.../pinot/ingestion/jobs}/SegmentUriPushJob.java | 31 ++--
.../ingestion}/utils/JobPreparationHelper.java | 20 +--
.../pinot/ingestion}/utils/PushLocation.java | 5 +-
.../pinot-spark}/README.md | 6 +-
.../pinot-spark}/pom.xml | 71 ++++++--
.../apache/pinot/spark/PinotSparkJobLauncher.java | 44 +++--
.../spark/jobs/SparkSegmentCreationFunction.java | 114 +++----------
.../pinot/spark/jobs/SparkSegmentCreationJob.java | 180 +++++++++++++++++++++
.../pinot/spark/jobs/SparkSegmentTarPushJob.java | 88 ++++++++++
.../pinot/spark/jobs/SparkSegmentUriPushJob.java | 71 ++++++++
.../utils/PinotSparkJobPreparationHelper.java | 36 +----
{pinot-connectors => pinot-ingestion-jobs}/pom.xml | 16 +-
...mentBuildPushOfflineClusterIntegrationTest.java | 16 +-
pom.xml | 14 +-
47 files changed, 1172 insertions(+), 605 deletions(-)
delete mode 100644 pinot-hadoop/README.md
copy {pinot-connectors/pinot-connector-kafka-base => pinot-ingestion-jobs/pinot-hadoop}/README.md (77%)
copy {pinot-hadoop => pinot-ingestion-jobs/pinot-hadoop}/pom.xml (96%)
copy {pinot-hadoop => pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/PinotHadoopJobLauncher.java (72%)
rename {pinot-hadoop => pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/io/CombineAvroKeyInputFormat.java (100%)
rename {pinot-hadoop => pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/io/FileHandler.java (97%)
rename {pinot-hadoop => pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/io/JsonPinotOutputFormat.java (100%)
rename {pinot-hadoop => pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/io/PinotOutputFormat.java (100%)
rename {pinot-hadoop => pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/io/PinotRecord.java (100%)
rename {pinot-hadoop => pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/io/PinotRecordSerialization.java (100%)
rename {pinot-hadoop => pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/io/PinotRecordWriter.java (98%)
rename pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentCreationJob.java => pinot-ingestion-jobs/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentCreationJob.java (57%)
rename pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/SegmentPreprocessingJob.java => pinot-ingestion-jobs/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/HadoopSegmentPreprocessingJob.java (77%)
rename {pinot-hadoop => pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/job/InternalConfigConstants.java (100%)
copy {pinot-hadoop => pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java (99%)
rename {pinot-hadoop => pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentPreprocessingMapper.java (94%)
rename {pinot-hadoop => pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/job/partitioners/GenericPartitioner.java (87%)
rename {pinot-hadoop => pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/job/partitioners/PartitionFunctionFactory.java (98%)
rename {pinot-hadoop => pinot-ingestion-jobs/pinot-hadoop}/src/main/java/org/apache/pinot/hadoop/job/reducers/SegmentPreprocessingReducer.java (99%)
copy pinot-hadoop/src/main/java/org/apache/pinot/hadoop/utils/JobPreparationHelper.java => pinot-ingestion-jobs/pinot-hadoop/src/main/java/org/apache/pinot/hadoop/utils/PinotHadoopJobPreparationHelper.java (64%)
rename {pinot-hadoop => pinot-ingestion-jobs/pinot-hadoop}/src/test/java/org/apache/pinot/hadoop/io/DeleteExtraPushedSegmentsTest.java (97%)
rename {pinot-hadoop => pinot-ingestion-jobs/pinot-hadoop}/src/test/java/org/apache/pinot/hadoop/io/PinotOutputFormatTest.java (100%)
rename {pinot-hadoop => pinot-ingestion-jobs/pinot-hadoop}/src/test/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationTest.java (100%)
copy {pinot-hadoop-filesystem => pinot-ingestion-jobs/pinot-ingestion-common}/pom.xml (70%)
rename {pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job => pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/common}/ControllerRestApi.java (96%)
rename {pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job => pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/common}/DefaultControllerRestApi.java (78%)
rename {pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job => pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/common}/JobConfigConstants.java (79%)
copy pinot-common/src/main/java/org/apache/pinot/common/exception/InvalidConfigException.java => pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/common/PinotIngestionJobType.java (79%)
rename {pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job => pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/jobs}/BaseSegmentJob.java (73%)
create mode 100644 pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/jobs/SegmentCreationJob.java
create mode 100644 pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/jobs/SegmentPreprocessingJob.java
rename {pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job => pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/jobs}/SegmentTarPushJob.java (73%)
rename {pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job => pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion/jobs}/SegmentUriPushJob.java (60%)
copy {pinot-hadoop/src/main/java/org/apache/pinot/hadoop => pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion}/utils/JobPreparationHelper.java (72%)
rename {pinot-hadoop/src/main/java/org/apache/pinot/hadoop => pinot-ingestion-jobs/pinot-ingestion-common/src/main/java/org/apache/pinot/ingestion}/utils/PushLocation.java (92%)
copy {pinot-connectors/pinot-connector-kafka-base => pinot-ingestion-jobs/pinot-spark}/README.md (77%)
rename {pinot-hadoop => pinot-ingestion-jobs/pinot-spark}/pom.xml (76%)
rename pinot-hadoop/src/main/java/org/apache/pinot/hadoop/PinotHadoopJobLauncher.java => pinot-ingestion-jobs/pinot-spark/src/main/java/org/apache/pinot/spark/PinotSparkJobLauncher.java (60%)
rename pinot-hadoop/src/main/java/org/apache/pinot/hadoop/job/mappers/SegmentCreationMapper.java => pinot-ingestion-jobs/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationFunction.java (76%)
create mode 100644 pinot-ingestion-jobs/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentCreationJob.java
create mode 100644 pinot-ingestion-jobs/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentTarPushJob.java
create mode 100644 pinot-ingestion-jobs/pinot-spark/src/main/java/org/apache/pinot/spark/jobs/SparkSegmentUriPushJob.java
rename pinot-hadoop/src/main/java/org/apache/pinot/hadoop/utils/JobPreparationHelper.java => pinot-ingestion-jobs/pinot-spark/src/main/java/org/apache/pinot/spark/utils/PinotSparkJobPreparationHelper.java (55%)
copy {pinot-connectors => pinot-ingestion-jobs}/pom.xml (84%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org