You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ne...@apache.org on 2019/06/01 00:38:20 UTC
[incubator-pinot] branch master updated: pinot-connector-kafka-0.9
module (#4252)
This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 00c2204 pinot-connector-kafka-0.9 module (#4252)
00c2204 is described below
commit 00c22042dcaee666b04c91e766246919b7571fd1
Author: Neha Pawar <np...@linkedin.com>
AuthorDate: Fri May 31 17:38:15 2019 -0700
pinot-connector-kafka-0.9 module (#4252)
Create a pinot-connectors module, and a sub module pinot-connector-kafka-0.9 to move the kafka 0.9 implementation
---
pinot-common/pom.xml | 18 ------
pinot-connectors/README.md | 23 ++++++++
.../pinot-connector-kafka-0.9/README.md | 24 ++++++++
pinot-connectors/pinot-connector-kafka-0.9/pom.xml | 67 ++++++++++++++++++++++
.../realtime/impl/kafka/ConsumerAndIterator.java | 0
.../impl/kafka/KafkaAvroMessageDecoder.java | 0
.../realtime/impl/kafka/KafkaBrokerWrapper.java | 0
.../impl/kafka/KafkaConnectionHandler.java | 0
.../realtime/impl/kafka/KafkaConsumerFactory.java | 0
.../realtime/impl/kafka/KafkaConsumerManager.java | 0
.../impl/kafka/KafkaHighLevelStreamConfig.java | 0
.../impl/kafka/KafkaJSONMessageDecoder.java | 0
.../impl/kafka/KafkaLowLevelStreamConfig.java | 0
.../impl/kafka/KafkaPartitionLevelConsumer.java | 0
.../impl/kafka/KafkaSimpleConsumerFactory.java | 0
.../impl/kafka/KafkaSimpleConsumerFactoryImpl.java | 0
.../realtime/impl/kafka}/KafkaStarterUtils.java | 3 +-
.../impl/kafka/KafkaStreamConfigProperties.java | 0
.../impl/kafka/KafkaStreamLevelConsumer.java | 0
.../impl/kafka/KafkaStreamMetadataProvider.java | 0
.../realtime/impl/kafka/SimpleConsumerFactory.java | 0
.../impl/kafka/SimpleConsumerMessageBatch.java | 0
.../impl/kafka/KafkaLowLevelStreamConfigTest.java | 0
.../kafka/KafkaPartitionLevelConsumerTest.java | 0
pinot-connectors/pom.xml | 63 ++++++++++++++++++++
pinot-core/pom.xml | 16 ------
.../AvroRecordToPinotRowGenerator.java | 0
.../kafka => stream}/SimpleAvroMessageDecoder.java | 0
.../pinot/core/realtime/stream/StreamConfig.java | 4 +-
.../AvroRecordToPinotRowGeneratorTest.java | 0
pinot-integration-tests/pom.xml | 6 ++
.../tests/BaseClusterIntegrationTest.java | 2 +-
.../ControllerPeriodicTasksIntegrationTests.java | 2 +-
.../tests/HybridClusterIntegrationTest.java | 3 +-
...ridClusterIntegrationTestCommandLineRunner.java | 2 +-
.../tests/RealtimeClusterIntegrationTest.java | 3 +-
pinot-perf/pom.xml | 6 ++
.../perf/BenchmarkRealtimeConsumptionSpeed.java | 2 +-
.../org/apache/pinot/perf/RealtimeStressTest.java | 2 +-
pinot-tools/pom.xml | 6 ++
.../org/apache/pinot/tools/HybridQuickstart.java | 2 +-
.../org/apache/pinot/tools/RealtimeQuickStart.java | 2 +-
.../tools/admin/command/StartKafkaCommand.java | 2 +-
.../admin/command/StreamAvroIntoKafkaCommand.java | 2 +-
.../pinot/tools/streams/AirlineDataStream.java | 2 +-
.../pinot/tools/streams/MeetupRsvpStream.java | 2 +-
pom.xml | 16 +-----
47 files changed, 213 insertions(+), 67 deletions(-)
diff --git a/pinot-common/pom.xml b/pinot-common/pom.xml
index 8f8a708..624da84 100644
--- a/pinot-common/pom.xml
+++ b/pinot-common/pom.xml
@@ -190,24 +190,6 @@
<artifactId>netty</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- </exclusion>
- <exclusion>
- <groupId>net.sf.jopt-simple</groupId>
- <artifactId>jopt-simple</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>net.sf.jopt-simple</groupId>
<artifactId>jopt-simple</artifactId>
</dependency>
diff --git a/pinot-connectors/README.md b/pinot-connectors/README.md
new file mode 100644
index 0000000..1c7dadf
--- /dev/null
+++ b/pinot-connectors/README.md
@@ -0,0 +1,23 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+# Pinot connectors module
+
+The pinot-connectors module is the place to write any pinot connectors to streams. For exampple, the pinot-connector-kafka-0.9 sub module contains the stream implementation for kafka-0.9. Dependencies to be shared across all sub modules are to be added in the pinot-connectors/pom.xml, and dependencies specific to the specific connector should go in its own pom file
diff --git a/pinot-connectors/pinot-connector-kafka-0.9/README.md b/pinot-connectors/pinot-connector-kafka-0.9/README.md
new file mode 100644
index 0000000..444713f
--- /dev/null
+++ b/pinot-connectors/pinot-connector-kafka-0.9/README.md
@@ -0,0 +1,24 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+# Pinot connector for kafka 0.9.x
+
+This is an implementation of the kafka stream for kafka versions 0.9.x. The version used in this implementation is kafka 0.9.0.1. This module compiles with version 0.9.0.0 as well, however we have not tested if it runs with the older versions.
+A stream plugin for another version of kafka, or another stream, can be added in a similar fashion. Refer to documentation on (Pluggable Streams)[https://pinot.readthedocs.io/en/latest/pluggable_streams.html] for the specfic interfaces to implement.
diff --git a/pinot-connectors/pinot-connector-kafka-0.9/pom.xml b/pinot-connectors/pinot-connector-kafka-0.9/pom.xml
new file mode 100644
index 0000000..ae0317e
--- /dev/null
+++ b/pinot-connectors/pinot-connector-kafka-0.9/pom.xml
@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>pinot-connectors</artifactId>
+ <groupId>org.apache.pinot</groupId>
+ <version>0.2.0-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>pinot-connector-kafka-0.9</artifactId>
+ <name>Pinot Connector Kafka 0.9</name>
+ <url>https://pinot.apache.org/</url>
+ <properties>
+ <pinot.root>${basedir}/../..</pinot.root>
+ <kafka.version>0.9.0.1</kafka.version>
+ </properties>
+
+ <dependencies>
+
+ <!-- Kafka -->
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.10</artifactId>
+ <version>${kafka.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.sf.jopt-simple</groupId>
+ <artifactId>jopt-simple</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/ConsumerAndIterator.java b/pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/ConsumerAndIterator.java
similarity index 100%
rename from pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/ConsumerAndIterator.java
rename to pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/ConsumerAndIterator.java
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaAvroMessageDecoder.java b/pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaAvroMessageDecoder.java
similarity index 100%
rename from pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaAvroMessageDecoder.java
rename to pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaAvroMessageDecoder.java
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaBrokerWrapper.java b/pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaBrokerWrapper.java
similarity index 100%
rename from pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaBrokerWrapper.java
rename to pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaBrokerWrapper.java
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaConnectionHandler.java b/pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaConnectionHandler.java
similarity index 100%
rename from pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaConnectionHandler.java
rename to pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaConnectionHandler.java
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaConsumerFactory.java b/pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaConsumerFactory.java
similarity index 100%
rename from pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaConsumerFactory.java
rename to pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaConsumerFactory.java
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaConsumerManager.java b/pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaConsumerManager.java
similarity index 100%
rename from pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaConsumerManager.java
rename to pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaConsumerManager.java
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaHighLevelStreamConfig.java b/pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaHighLevelStreamConfig.java
similarity index 100%
rename from pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaHighLevelStreamConfig.java
rename to pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaHighLevelStreamConfig.java
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaJSONMessageDecoder.java b/pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaJSONMessageDecoder.java
similarity index 100%
rename from pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaJSONMessageDecoder.java
rename to pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaJSONMessageDecoder.java
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaLowLevelStreamConfig.java b/pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaLowLevelStreamConfig.java
similarity index 100%
rename from pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaLowLevelStreamConfig.java
rename to pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaLowLevelStreamConfig.java
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaPartitionLevelConsumer.java b/pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaPartitionLevelConsumer.java
similarity index 100%
rename from pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaPartitionLevelConsumer.java
rename to pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaPartitionLevelConsumer.java
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaSimpleConsumerFactory.java b/pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaSimpleConsumerFactory.java
similarity index 100%
rename from pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaSimpleConsumerFactory.java
rename to pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaSimpleConsumerFactory.java
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaSimpleConsumerFactoryImpl.java b/pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaSimpleConsumerFactoryImpl.java
similarity index 100%
rename from pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaSimpleConsumerFactoryImpl.java
rename to pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaSimpleConsumerFactoryImpl.java
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/KafkaStarterUtils.java b/pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaStarterUtils.java
similarity index 98%
rename from pinot-common/src/main/java/org/apache/pinot/common/utils/KafkaStarterUtils.java
rename to pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaStarterUtils.java
index 0b4e6ee..5f1de99 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/KafkaStarterUtils.java
+++ b/pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaStarterUtils.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.common.utils;
+package org.apache.pinot.core.realtime.impl.kafka;
import java.io.File;
import java.security.Permission;
@@ -28,6 +28,7 @@ import kafka.server.KafkaConfig;
import kafka.server.KafkaServerStartable;
import org.I0Itec.zkclient.ZkClient;
import org.apache.commons.io.FileUtils;
+import org.apache.pinot.common.utils.ZkStarter;
/**
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaStreamConfigProperties.java b/pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaStreamConfigProperties.java
similarity index 100%
rename from pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaStreamConfigProperties.java
rename to pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaStreamConfigProperties.java
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaStreamLevelConsumer.java b/pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaStreamLevelConsumer.java
similarity index 100%
rename from pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaStreamLevelConsumer.java
rename to pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaStreamLevelConsumer.java
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaStreamMetadataProvider.java b/pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaStreamMetadataProvider.java
similarity index 100%
rename from pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaStreamMetadataProvider.java
rename to pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/KafkaStreamMetadataProvider.java
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/SimpleConsumerFactory.java b/pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/SimpleConsumerFactory.java
similarity index 100%
rename from pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/SimpleConsumerFactory.java
rename to pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/SimpleConsumerFactory.java
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/SimpleConsumerMessageBatch.java b/pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/SimpleConsumerMessageBatch.java
similarity index 100%
rename from pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/SimpleConsumerMessageBatch.java
rename to pinot-connectors/pinot-connector-kafka-0.9/src/main/java/org/apache/pinot/core/realtime/impl/kafka/SimpleConsumerMessageBatch.java
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/realtime/impl/kafka/KafkaLowLevelStreamConfigTest.java b/pinot-connectors/pinot-connector-kafka-0.9/src/test/java/org/apache/pinot/core/realtime/impl/kafka/KafkaLowLevelStreamConfigTest.java
similarity index 100%
rename from pinot-core/src/test/java/org/apache/pinot/core/realtime/impl/kafka/KafkaLowLevelStreamConfigTest.java
rename to pinot-connectors/pinot-connector-kafka-0.9/src/test/java/org/apache/pinot/core/realtime/impl/kafka/KafkaLowLevelStreamConfigTest.java
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/realtime/kafka/KafkaPartitionLevelConsumerTest.java b/pinot-connectors/pinot-connector-kafka-0.9/src/test/java/org/apache/pinot/core/realtime/impl/kafka/KafkaPartitionLevelConsumerTest.java
similarity index 100%
rename from pinot-core/src/test/java/org/apache/pinot/core/realtime/kafka/KafkaPartitionLevelConsumerTest.java
rename to pinot-connectors/pinot-connector-kafka-0.9/src/test/java/org/apache/pinot/core/realtime/impl/kafka/KafkaPartitionLevelConsumerTest.java
diff --git a/pinot-connectors/pom.xml b/pinot-connectors/pom.xml
new file mode 100644
index 0000000..3695189
--- /dev/null
+++ b/pinot-connectors/pom.xml
@@ -0,0 +1,63 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>pinot</artifactId>
+ <groupId>org.apache.pinot</groupId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>pinot-connectors</artifactId>
+ <packaging>pom</packaging>
+ <name>Pinot Connectors</name>
+ <properties>
+ <pinot.root>${basedir}/..</pinot.root>
+ </properties>
+
+ <modules>
+ <module>pinot-connector-kafka-0.9</module>
+ </modules>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.pinot</groupId>
+ <artifactId>pinot-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.pinot</groupId>
+ <artifactId>pinot-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- test -->
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+</project>
\ No newline at end of file
diff --git a/pinot-core/pom.xml b/pinot-core/pom.xml
index eb6fd83..51c9231 100644
--- a/pinot-core/pom.xml
+++ b/pinot-core/pom.xml
@@ -169,22 +169,6 @@
<artifactId>jopt-simple</artifactId>
</dependency>
- <!-- Kafka -->
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
- <exclusions>
- <exclusion>
- <groupId>net.sf.jopt-simple</groupId>
- <artifactId>jopt-simple</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
<!-- test -->
<dependency>
<groupId>org.apache.pinot</groupId>
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/AvroRecordToPinotRowGenerator.java b/pinot-core/src/main/java/org/apache/pinot/core/realtime/stream/AvroRecordToPinotRowGenerator.java
similarity index 100%
rename from pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/AvroRecordToPinotRowGenerator.java
rename to pinot-core/src/main/java/org/apache/pinot/core/realtime/stream/AvroRecordToPinotRowGenerator.java
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/SimpleAvroMessageDecoder.java b/pinot-core/src/main/java/org/apache/pinot/core/realtime/stream/SimpleAvroMessageDecoder.java
similarity index 100%
rename from pinot-core/src/main/java/org/apache/pinot/core/realtime/impl/kafka/SimpleAvroMessageDecoder.java
rename to pinot-core/src/main/java/org/apache/pinot/core/realtime/stream/SimpleAvroMessageDecoder.java
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/realtime/stream/StreamConfig.java b/pinot-core/src/main/java/org/apache/pinot/core/realtime/stream/StreamConfig.java
index b25b371..1d37534 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/realtime/stream/StreamConfig.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/realtime/stream/StreamConfig.java
@@ -27,7 +27,6 @@ import java.util.concurrent.TimeUnit;
import org.apache.pinot.common.utils.DataSize;
import org.apache.pinot.common.utils.EqualityUtils;
import org.apache.pinot.common.utils.time.TimeUtils;
-import org.apache.pinot.core.realtime.impl.kafka.KafkaConsumerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,7 +47,8 @@ public class StreamConfig {
private static final int DEFAULT_FLUSH_THRESHOLD_ROWS = 5_000_000;
private static final long DEFAULT_FLUSH_THRESHOLD_TIME = TimeUnit.MILLISECONDS.convert(6, TimeUnit.HOURS);
private static final long DEFAULT_DESIRED_SEGMENT_SIZE_BYTES = 200 * 1024 * 1024; // 200M
- private static final String DEFAULT_CONSUMER_FACTORY_CLASS_NAME_STRING = KafkaConsumerFactory.class.getName();
+ private static final String DEFAULT_CONSUMER_FACTORY_CLASS_NAME_STRING =
+ "org.apache.pinot.core.realtime.impl.kafka.KafkaConsumerFactory";
protected static final long DEFAULT_STREAM_CONNECTION_TIMEOUT_MILLIS = 30_000;
protected static final int DEFAULT_STREAM_FETCH_TIMEOUT_MILLIS = 5_000;
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/realtime/impl/kafka/AvroRecordToPinotRowGeneratorTest.java b/pinot-core/src/test/java/org/apache/pinot/core/realtime/stream/AvroRecordToPinotRowGeneratorTest.java
similarity index 100%
rename from pinot-core/src/test/java/org/apache/pinot/core/realtime/impl/kafka/AvroRecordToPinotRowGeneratorTest.java
rename to pinot-core/src/test/java/org/apache/pinot/core/realtime/stream/AvroRecordToPinotRowGeneratorTest.java
diff --git a/pinot-integration-tests/pom.xml b/pinot-integration-tests/pom.xml
index 20ad7c8..5467174 100644
--- a/pinot-integration-tests/pom.xml
+++ b/pinot-integration-tests/pom.xml
@@ -189,6 +189,12 @@
<version>${project.version}</version>
<type>test-jar</type>
</dependency>
+ <!-- TODO: figure out a way to avoid explicitly setting the kafka module dependency, and instead inject it, such that any kafka stream implementation can use this -->
+ <dependency>
+ <groupId>org.apache.pinot</groupId>
+ <artifactId>pinot-connector-kafka-0.9</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTest.java
index fed0b68..21c2140 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTest.java
@@ -33,10 +33,10 @@ import org.apache.commons.io.FileUtils;
import org.apache.pinot.client.ConnectionFactory;
import org.apache.pinot.common.config.TableTaskConfig;
import org.apache.pinot.common.config.TagNameUtils;
-import org.apache.pinot.common.utils.KafkaStarterUtils;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
import org.apache.pinot.common.utils.ZkStarter;
import org.apache.pinot.core.realtime.impl.kafka.KafkaConsumerFactory;
+import org.apache.pinot.core.realtime.impl.kafka.KafkaStarterUtils;
import org.apache.pinot.util.TestUtils;
import org.testng.Assert;
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ControllerPeriodicTasksIntegrationTests.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ControllerPeriodicTasksIntegrationTests.java
index 77ca404..a947373 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ControllerPeriodicTasksIntegrationTests.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ControllerPeriodicTasksIntegrationTests.java
@@ -46,11 +46,11 @@ import org.apache.pinot.common.metrics.ControllerGauge;
import org.apache.pinot.common.metrics.ControllerMeter;
import org.apache.pinot.common.metrics.ControllerMetrics;
import org.apache.pinot.common.utils.CommonConstants;
-import org.apache.pinot.common.utils.KafkaStarterUtils;
import org.apache.pinot.common.utils.helix.HelixHelper;
import org.apache.pinot.common.utils.retry.RetryPolicies;
import org.apache.pinot.controller.ControllerConf;
import org.apache.pinot.core.indexsegment.generator.SegmentVersion;
+import org.apache.pinot.core.realtime.impl.kafka.KafkaStarterUtils;
import org.apache.pinot.util.TestUtils;
import org.testng.Assert;
import org.testng.ITestContext;
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/HybridClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/HybridClusterIntegrationTest.java
index fa88396..f9ca74a 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/HybridClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/HybridClusterIntegrationTest.java
@@ -27,14 +27,13 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
-import kafka.server.KafkaServerStartable;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.config.TableNameBuilder;
import org.apache.pinot.common.data.Schema;
import org.apache.pinot.common.utils.CommonConstants;
import org.apache.pinot.common.utils.JsonUtils;
-import org.apache.pinot.common.utils.KafkaStarterUtils;
import org.apache.pinot.controller.ControllerConf;
+import org.apache.pinot.core.realtime.impl.kafka.KafkaStarterUtils;
import org.apache.pinot.util.TestUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/HybridClusterIntegrationTestCommandLineRunner.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/HybridClusterIntegrationTestCommandLineRunner.java
index ed1cc47..6e5474f 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/HybridClusterIntegrationTestCommandLineRunner.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/HybridClusterIntegrationTestCommandLineRunner.java
@@ -38,8 +38,8 @@ import kafka.server.KafkaServerStartable;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.data.Schema;
import org.apache.pinot.common.utils.JsonUtils;
-import org.apache.pinot.common.utils.KafkaStarterUtils;
import org.apache.pinot.controller.ControllerConf;
+import org.apache.pinot.core.realtime.impl.kafka.KafkaStarterUtils;
import org.apache.pinot.tools.query.comparison.QueryComparison;
import org.apache.pinot.util.TestUtils;
import org.testng.Assert;
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/RealtimeClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/RealtimeClusterIntegrationTest.java
index 37df2e4..d5196bf 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/RealtimeClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/RealtimeClusterIntegrationTest.java
@@ -24,10 +24,9 @@ import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
-import kafka.server.KafkaServerStartable;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.data.Schema;
-import org.apache.pinot.common.utils.KafkaStarterUtils;
+import org.apache.pinot.core.realtime.impl.kafka.KafkaStarterUtils;
import org.apache.pinot.util.TestUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
diff --git a/pinot-perf/pom.xml b/pinot-perf/pom.xml
index 7a69487..a37057b 100644
--- a/pinot-perf/pom.xml
+++ b/pinot-perf/pom.xml
@@ -50,6 +50,12 @@
<groupId>org.apache.pinot</groupId>
<artifactId>pinot-broker</artifactId>
</dependency>
+ <!-- TODO: figure out a way to avoid explicitly setting the kafka module dependency, and instead inject it, such that any kafka stream implementation can use this -->
+ <dependency>
+ <groupId>org.apache.pinot</groupId>
+ <artifactId>pinot-connector-kafka-0.9</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.pinot</groupId>
<artifactId>pinot-integration-tests</artifactId>
diff --git a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkRealtimeConsumptionSpeed.java b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkRealtimeConsumptionSpeed.java
index faa8583..8bbb4d5 100644
--- a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkRealtimeConsumptionSpeed.java
+++ b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkRealtimeConsumptionSpeed.java
@@ -26,8 +26,8 @@ import java.util.List;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import kafka.server.KafkaServerStartable;
-import org.apache.pinot.common.utils.KafkaStarterUtils;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
+import org.apache.pinot.core.realtime.impl.kafka.KafkaStarterUtils;
import org.apache.pinot.integration.tests.ClusterIntegrationTestUtils;
import org.apache.pinot.integration.tests.RealtimeClusterIntegrationTest;
import org.apache.pinot.util.TestUtils;
diff --git a/pinot-perf/src/main/java/org/apache/pinot/perf/RealtimeStressTest.java b/pinot-perf/src/main/java/org/apache/pinot/perf/RealtimeStressTest.java
index 50a0e3f..71d28e7 100644
--- a/pinot-perf/src/main/java/org/apache/pinot/perf/RealtimeStressTest.java
+++ b/pinot-perf/src/main/java/org/apache/pinot/perf/RealtimeStressTest.java
@@ -26,8 +26,8 @@ import java.util.List;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import kafka.server.KafkaServerStartable;
-import org.apache.pinot.common.utils.KafkaStarterUtils;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
+import org.apache.pinot.core.realtime.impl.kafka.KafkaStarterUtils;
import org.apache.pinot.integration.tests.ClusterIntegrationTestUtils;
import org.apache.pinot.integration.tests.OfflineClusterIntegrationTest;
import org.apache.pinot.integration.tests.RealtimeClusterIntegrationTest;
diff --git a/pinot-tools/pom.xml b/pinot-tools/pom.xml
index 766852c..84a88a2 100644
--- a/pinot-tools/pom.xml
+++ b/pinot-tools/pom.xml
@@ -54,6 +54,12 @@
<groupId>org.apache.pinot</groupId>
<artifactId>pinot-broker</artifactId>
</dependency>
+ <!-- TODO: figure out a way to avoid explicitly setting the kafka module dependency, and instead inject it, such that any kafka stream implementation can use this -->
+ <dependency>
+ <groupId>org.apache.pinot</groupId>
+ <artifactId>pinot-connector-kafka-0.9</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
index 5328f9c..fe004d3 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
@@ -26,9 +26,9 @@ import java.net.URL;
import kafka.server.KafkaServerStartable;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.data.Schema;
-import org.apache.pinot.common.utils.KafkaStarterUtils;
import org.apache.pinot.common.utils.ZkStarter;
import org.apache.pinot.core.data.readers.FileFormat;
+import org.apache.pinot.core.realtime.impl.kafka.KafkaStarterUtils;
import org.apache.pinot.tools.Quickstart.Color;
import org.apache.pinot.tools.admin.command.QuickstartRunner;
import org.apache.pinot.tools.streams.AirlineDataStream;
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java
index aaea53b..8df1fbb 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java
@@ -24,8 +24,8 @@ import java.io.File;
import java.net.URL;
import kafka.server.KafkaServerStartable;
import org.apache.commons.io.FileUtils;
-import org.apache.pinot.common.utils.KafkaStarterUtils;
import org.apache.pinot.common.utils.ZkStarter;
+import org.apache.pinot.core.realtime.impl.kafka.KafkaStarterUtils;
import org.apache.pinot.tools.Quickstart.Color;
import org.apache.pinot.tools.admin.command.QuickstartRunner;
import org.apache.pinot.tools.streams.MeetupRsvpStream;
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/StartKafkaCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/StartKafkaCommand.java
index 638752f..38f042c 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/StartKafkaCommand.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/StartKafkaCommand.java
@@ -20,7 +20,7 @@ package org.apache.pinot.tools.admin.command;
import java.io.File;
import java.io.IOException;
-import org.apache.pinot.common.utils.KafkaStarterUtils;
+import org.apache.pinot.core.realtime.impl.kafka.KafkaStarterUtils;
import org.apache.pinot.tools.Command;
import org.kohsuke.args4j.Option;
import org.slf4j.Logger;
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/StreamAvroIntoKafkaCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/StreamAvroIntoKafkaCommand.java
index e7ba939..0a75023 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/StreamAvroIntoKafkaCommand.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/StreamAvroIntoKafkaCommand.java
@@ -30,7 +30,7 @@ import kafka.producer.ProducerConfig;
import org.apache.avro.file.DataFileStream;
import org.apache.avro.generic.GenericRecord;
import org.apache.pinot.common.utils.HashUtil;
-import org.apache.pinot.common.utils.KafkaStarterUtils;
+import org.apache.pinot.core.realtime.impl.kafka.KafkaStarterUtils;
import org.apache.pinot.core.util.AvroUtils;
import org.apache.pinot.tools.Command;
import org.kohsuke.args4j.Option;
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/streams/AirlineDataStream.java b/pinot-tools/src/main/java/org/apache/pinot/tools/streams/AirlineDataStream.java
index 621e128..7c238bf 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/streams/AirlineDataStream.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/streams/AirlineDataStream.java
@@ -37,7 +37,7 @@ import org.apache.pinot.common.data.FieldSpec;
import org.apache.pinot.common.data.Schema;
import org.apache.pinot.common.data.TimeFieldSpec;
import org.apache.pinot.common.utils.JsonUtils;
-import org.apache.pinot.common.utils.KafkaStarterUtils;
+import org.apache.pinot.core.realtime.impl.kafka.KafkaStarterUtils;
import org.apache.pinot.tools.Quickstart;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/streams/MeetupRsvpStream.java b/pinot-tools/src/main/java/org/apache/pinot/tools/streams/MeetupRsvpStream.java
index 2218ea8..114072f 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/streams/MeetupRsvpStream.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/streams/MeetupRsvpStream.java
@@ -35,8 +35,8 @@ import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;
import org.apache.pinot.common.data.Schema;
import org.apache.pinot.common.utils.JsonUtils;
-import org.apache.pinot.common.utils.KafkaStarterUtils;
import org.apache.pinot.core.realtime.impl.kafka.KafkaJSONMessageDecoder;
+import org.apache.pinot.core.realtime.impl.kafka.KafkaStarterUtils;
import org.glassfish.tyrus.client.ClientManager;
diff --git a/pom.xml b/pom.xml
index 83ed620..930fb6b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -55,6 +55,7 @@
<module>pinot-azure-filesystem</module>
<module>pinot-orc</module>
<module>pinot-parquet</module>
+ <module>pinot-connectors</module>
</modules>
<licenses>
@@ -118,8 +119,6 @@
<avro.version>1.7.6</avro.version>
<parquet.version>1.8.0</parquet.version>
<helix.version>0.8.2</helix.version>
- <!-- jfim: for Kafka 0.9.0.0, use zkclient 0.7 -->
- <kafka.version>0.9.0.1</kafka.version>
<zkclient.version>0.7</zkclient.version>
<jackson.version>2.9.8</jackson.version>
<async-http-client.version>1.9.21</async-http-client.version>
@@ -585,19 +584,6 @@
<version>${jackson.version}</version>
</dependency>
- <!-- Kafka -->
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
- <version>${kafka.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
<!-- Hadoop -->
<dependency>
<groupId>org.apache.hadoop</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org