You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by rz...@apache.org on 2023/10/19 07:25:43 UTC
[storm] 12/13: STORM-3988 - Remove "storm-sql-mongodb"
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch STORM-3988
in repository https://gitbox.apache.org/repos/asf/storm.git
commit 4dd83d7b84affed2b3db2106f630e4773d285b1d
Author: Richard Zowalla <ri...@hs-heilbronn.de>
AuthorDate: Thu Oct 19 09:09:27 2023 +0200
STORM-3988 - Remove "storm-sql-mongodb"
---
pom.xml | 5 -
sql/pom.xml | 1 -
sql/storm-sql-external/storm-sql-mongodb/pom.xml | 93 ---------------
.../sql/mongodb/MongoDataSourcesProvider.java | 125 ---------------------
...rg.apache.storm.sql.runtime.DataSourcesProvider | 16 ---
.../sql/mongodb/TestMongoDataSourcesProvider.java | 57 ----------
6 files changed, 297 deletions(-)
diff --git a/pom.xml b/pom.xml
index 3b3d2d31c..06fe85079 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,15 +126,10 @@
<awaitility.version>3.1.0</awaitility.version>
<hdrhistogram.version>2.1.10</hdrhistogram.version>
<hamcrest.version>2.2</hamcrest.version>
- <cassandra.version>2.1.7</cassandra.version>
<elasticsearch.version>5.2.2</elasticsearch.version>
<calcite.version>1.16.0</calcite.version>
- <mongodb.version>3.2.0</mongodb.version>
- <solr.version>5.2.1</solr.version>
- <jpmml.version>1.0.22</jpmml.version>
<jedis.version>2.9.0</jedis.version>
<activemq.version>5.18.2</activemq.version>
- <rocketmq.version>4.2.0</rocketmq.version>
<jackson.version>2.15.2</jackson.version>
<jackson.databind.version>2.15.2</jackson.databind.version>
diff --git a/sql/pom.xml b/sql/pom.xml
index a14665f4c..cfc181cc6 100644
--- a/sql/pom.xml
+++ b/sql/pom.xml
@@ -41,7 +41,6 @@
<module>storm-sql-runtime</module>
<module>storm-sql-external/storm-sql-kafka</module>
<module>storm-sql-external/storm-sql-redis</module>
- <module>storm-sql-external/storm-sql-mongodb</module>
<module>storm-sql-external/storm-sql-hdfs</module>
</modules>
diff --git a/sql/storm-sql-external/storm-sql-mongodb/pom.xml b/sql/storm-sql-external/storm-sql-mongodb/pom.xml
deleted file mode 100644
index 40bb72388..000000000
--- a/sql/storm-sql-external/storm-sql-mongodb/pom.xml
+++ /dev/null
@@ -1,93 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <artifactId>storm</artifactId>
- <groupId>org.apache.storm</groupId>
- <version>2.6.0-SNAPSHOT</version>
- <relativePath>../../../pom.xml</relativePath>
- </parent>
-
- <artifactId>storm-sql-mongodb</artifactId>
-
- <developers>
- <developer>
- <id>vesense</id>
- <name>Xin Wang</name>
- <email>xinwang@apache.org</email>
- </developer>
- </developers>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-client</artifactId>
- <version>${project.version}</version>
- <scope>${provided.scope}</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-sql-runtime</artifactId>
- <version>${project.version}</version>
- <scope>${provided.scope}</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-sql-runtime</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.storm</groupId>
- <artifactId>storm-mongodb</artifactId>
- <version>${project.version}</version>
- <scope>${provided.scope}</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest</artifactId>
- </dependency>
- </dependencies>
- <build>
- <sourceDirectory>src/jvm</sourceDirectory>
- <testSourceDirectory>src/test</testSourceDirectory>
- <resources>
- <resource>
- <directory>${basedir}/src/resources</directory>
- </resource>
- </resources>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <!--Note - the version would be inherited-->
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-pmd-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/sql/storm-sql-external/storm-sql-mongodb/src/jvm/org/apache/storm/sql/mongodb/MongoDataSourcesProvider.java b/sql/storm-sql-external/storm-sql-mongodb/src/jvm/org/apache/storm/sql/mongodb/MongoDataSourcesProvider.java
deleted file mode 100644
index 827cbdb2e..000000000
--- a/sql/storm-sql-external/storm-sql-mongodb/src/jvm/org/apache/storm/sql/mongodb/MongoDataSourcesProvider.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.storm.sql.mongodb;
-
-import com.google.common.base.Preconditions;
-
-import java.net.URI;
-import java.util.List;
-import java.util.Properties;
-
-import org.apache.storm.mongodb.bolt.MongoInsertBolt;
-import org.apache.storm.mongodb.common.mapper.MongoMapper;
-import org.apache.storm.sql.runtime.DataSourcesProvider;
-import org.apache.storm.sql.runtime.FieldInfo;
-import org.apache.storm.sql.runtime.IOutputSerializer;
-import org.apache.storm.sql.runtime.ISqlStreamsDataSource;
-import org.apache.storm.sql.runtime.utils.FieldInfoUtils;
-import org.apache.storm.sql.runtime.utils.SerdeUtils;
-import org.apache.storm.topology.IRichBolt;
-import org.apache.storm.topology.IRichSpout;
-import org.apache.storm.tuple.ITuple;
-import org.apache.storm.tuple.Values;
-
-import org.bson.Document;
-
-/**
- * Create a MongoDB sink based on the URI and properties. The URI has the format of
- * mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]].
- * The properties are in JSON format which specifies the name of the MongoDB collection and etc.
- */
-public class MongoDataSourcesProvider implements DataSourcesProvider {
- public static final String SCHEME_NAME = "mongodb";
- public static final String VALUE_SERIALIZED_FIELD = "ser.field";
- public static final String TRIDENT_VALUE_SERIALIZED_FIELD = "trident.ser.field";
- public static final String DEFAULT_VALUE_SERIALIZED_FIELD = "tridentSerField";
- public static final String COLLECTION_NAME = "collection.name";
-
- private static class MongoStreamsDataSource implements ISqlStreamsDataSource {
- private final String url;
- private final Properties props;
- private final IOutputSerializer serializer;
-
- private MongoStreamsDataSource(String url, Properties props, IOutputSerializer serializer) {
- this.url = url;
- this.props = props;
- this.serializer = serializer;
- }
-
- @Override
- public IRichSpout getProducer() {
- throw new UnsupportedOperationException(this.getClass().getName() + " doesn't provide Producer");
- }
-
- @Override
- public IRichBolt getConsumer() {
- Preconditions.checkArgument(!props.isEmpty(), "Writable MongoDB must contain collection config");
- String serField;
- if (props.contains(VALUE_SERIALIZED_FIELD)) {
- serField = props.getProperty(VALUE_SERIALIZED_FIELD);
- } else if (props.contains(TRIDENT_VALUE_SERIALIZED_FIELD)) {
- // backward compatibility
- serField = props.getProperty(TRIDENT_VALUE_SERIALIZED_FIELD);
- } else {
- serField = DEFAULT_VALUE_SERIALIZED_FIELD;
- }
-
- MongoMapper mapper = new SqlMongoMapper(serField, serializer);
- return new MongoInsertBolt(url, props.getProperty(COLLECTION_NAME), mapper);
- }
- }
-
- private static class SqlMongoMapper implements MongoMapper {
- private final String serField;
- private final IOutputSerializer serializer;
-
- private SqlMongoMapper(String serField, IOutputSerializer serializer) {
- this.serField = serField;
- this.serializer = serializer;
- }
-
- @Override
- public Document toDocument(ITuple tuple) {
- Document document = new Document();
- Values values = (Values) tuple.getValue(1);
- byte[] array = serializer.write(values, null).array();
- document.append(serField, array);
- return document;
- }
-
- @Override
- public Document toDocumentByKeys(List<Object> keys) {
- return null;
- }
- }
-
- @Override
- public String scheme() {
- return SCHEME_NAME;
- }
-
- @Override
- public ISqlStreamsDataSource constructStreams(URI uri, String inputFormatClass, String outputFormatClass,
- Properties properties, List<FieldInfo> fields) {
- List<String> fieldNames = FieldInfoUtils.getFieldNames(fields);
- IOutputSerializer serializer = SerdeUtils.getSerializer(outputFormatClass, properties, fieldNames);
- return new MongoStreamsDataSource(uri.toString(), properties, serializer);
- }
-
-}
diff --git a/sql/storm-sql-external/storm-sql-mongodb/src/resources/META-INF/services/org.apache.storm.sql.runtime.DataSourcesProvider b/sql/storm-sql-external/storm-sql-mongodb/src/resources/META-INF/services/org.apache.storm.sql.runtime.DataSourcesProvider
deleted file mode 100644
index e46d794b7..000000000
--- a/sql/storm-sql-external/storm-sql-mongodb/src/resources/META-INF/services/org.apache.storm.sql.runtime.DataSourcesProvider
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-org.apache.storm.sql.mongodb.MongoDataSourcesProvider
\ No newline at end of file
diff --git a/sql/storm-sql-external/storm-sql-mongodb/src/test/org/apache/storm/sql/mongodb/TestMongoDataSourcesProvider.java b/sql/storm-sql-external/storm-sql-mongodb/src/test/org/apache/storm/sql/mongodb/TestMongoDataSourcesProvider.java
deleted file mode 100644
index d6c730985..000000000
--- a/sql/storm-sql-external/storm-sql-mongodb/src/test/org/apache/storm/sql/mongodb/TestMongoDataSourcesProvider.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.storm.sql.mongodb;
-
-import java.net.URI;
-import java.util.List;
-import java.util.Properties;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.storm.mongodb.bolt.MongoInsertBolt;
-import org.apache.storm.sql.runtime.DataSourcesRegistry;
-import org.apache.storm.sql.runtime.FieldInfo;
-import org.apache.storm.sql.runtime.ISqlStreamsDataSource;
-import org.apache.storm.topology.IRichBolt;
-import org.junit.jupiter.api.Test;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-
-public class TestMongoDataSourcesProvider {
-
- private static final List<FieldInfo> FIELDS = ImmutableList.of(
- new FieldInfo("ID", int.class, true),
- new FieldInfo("val", String.class, false));
- private static final Properties TBL_PROPERTIES = new Properties();
-
- static {
- TBL_PROPERTIES.put("collection.name", "collection1");
- TBL_PROPERTIES.put("ser.field", "serField");
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void testMongoSink() throws Exception {
- ISqlStreamsDataSource ds = DataSourcesRegistry.constructStreamsDataSource(
- URI.create("mongodb://127.0.0.1:27017/test"), null, null, TBL_PROPERTIES, FIELDS);
- assertNotNull(ds);
-
- IRichBolt consumer = ds.getConsumer();
- assertEquals(MongoInsertBolt.class, consumer.getClass());
- }
-}