You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2016/09/26 21:52:46 UTC
[1/3] storm git commit: STORM-2089 Replace Consumer of
ISqlTridentDataSource with SqlTridentConsumer
Repository: storm
Updated Branches:
refs/heads/1.x-branch 8bcdd7b5c -> 8fbef3793
STORM-2089 Replace Consumer of ISqlTridentDataSource with SqlTridentConsumer
* SqlTridentConsumer contains StateFactory and StateUpdater which is needed to store tuples to State via batch
* Apply the change to storm-sql-kafka
* move out JsonScheme and JsonSerializer to runtime
* it will be used from other external sql modules
* add javadoc to ISqlTridentDataSource
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/130b0514
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/130b0514
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/130b0514
Branch: refs/heads/1.x-branch
Commit: 130b0514dd8dd6ec382bc274da98f317dbb61315
Parents: 8bcdd7b
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Sep 13 11:40:13 2016 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Sep 27 06:50:25 2016 +0900
----------------------------------------------------------------------
.../trident/TridentLogicalPlanCompiler.java | 9 +-
.../backends/trident/TestPlanCompiler.java | 33 +++--
.../org/apache/storm/sql/kafka/JsonScheme.java | 58 --------
.../apache/storm/sql/kafka/JsonSerializer.java | 56 --------
.../sql/kafka/KafkaDataSourcesProvider.java | 61 +++-----
.../storm/sql/kafka/TestJsonRepresentation.java | 50 -------
.../sql/kafka/TestKafkaDataSourcesProvider.java | 26 +++-
external/sql/storm-sql-runtime/pom.xml | 4 +
.../org/apache/storm/sql/runtime/FieldInfo.java | 4 +-
.../sql/runtime/ISqlTridentDataSource.java | 41 +++++-
.../sql/runtime/SimpleSqlTridentConsumer.java | 24 ++++
.../sql/runtime/serde/json/JsonScheme.java | 58 ++++++++
.../sql/runtime/serde/json/JsonSerializer.java | 57 ++++++++
.../storm/sql/TestJsonRepresentation.java | 52 +++++++
.../test/org/apache/storm/sql/TestUtils.java | 138 +++++++++----------
15 files changed, 367 insertions(+), 304 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-core/src/jvm/org/apache/storm/sql/compiler/backends/trident/TridentLogicalPlanCompiler.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-core/src/jvm/org/apache/storm/sql/compiler/backends/trident/TridentLogicalPlanCompiler.java b/external/sql/storm-sql-core/src/jvm/org/apache/storm/sql/compiler/backends/trident/TridentLogicalPlanCompiler.java
index 688b164..a0bfa21 100644
--- a/external/sql/storm-sql-core/src/jvm/org/apache/storm/sql/compiler/backends/trident/TridentLogicalPlanCompiler.java
+++ b/external/sql/storm-sql-core/src/jvm/org/apache/storm/sql/compiler/backends/trident/TridentLogicalPlanCompiler.java
@@ -131,8 +131,13 @@ public class TridentLogicalPlanCompiler extends PostOrderRelNodeVisitor<IAggrega
List<String> inputFields = input.getRowType().getFieldNames();
List<String> outputFields = modify.getRowType().getFieldNames();
- return inputStream.each(new Fields(inputFields), sources.get(tableName).getConsumer(), new Fields(outputFields))
- .name(stageName);
+ ISqlTridentDataSource.SqlTridentConsumer consumer = sources.get(tableName).getConsumer();
+
+ // In fact this is normally the end of stream, but to match the return type we open new streams based on State values
+ return inputStream
+ .partitionPersist(consumer.getStateFactory(), new Fields(inputFields), consumer.getStateUpdater(),
+ new Fields(outputFields))
+ .newValuesStream().name(stageName);
}
@Override
http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/trident/TestPlanCompiler.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/trident/TestPlanCompiler.java b/external/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/trident/TestPlanCompiler.java
index 35d1364..85510a0 100644
--- a/external/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/trident/TestPlanCompiler.java
+++ b/external/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/trident/TestPlanCompiler.java
@@ -29,7 +29,6 @@ import org.apache.storm.ILocalCluster;
import org.apache.storm.LocalCluster;
import org.apache.storm.generated.StormTopology;
import org.apache.storm.sql.TestUtils;
-import org.apache.storm.sql.TestUtils.MockSqlTridentDataSource.CollectDataFunction;
import org.apache.storm.sql.compiler.TestCompilerUtils;
import org.apache.storm.sql.runtime.ISqlTridentDataSource;
import org.apache.storm.sql.runtime.trident.AbstractTridentProcessor;
@@ -46,7 +45,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
-import static org.apache.storm.sql.TestUtils.MockSqlTridentDataSource.CollectDataFunction.getCollectedValues;
+import static org.apache.storm.sql.TestUtils.MockState.getCollectedValues;
public class TestPlanCompiler {
private final JavaTypeFactory typeFactory = new JavaTypeFactoryImpl(
@@ -68,7 +67,8 @@ public class TestPlanCompiler {
final AbstractTridentProcessor proc = compiler.compileForTest(state.tree());
final TridentTopology topo = proc.build(data);
Fields f = proc.outputStream().getOutputFields();
- proc.outputStream().each(f, new CollectDataFunction(), new Fields()).toStream();
+ proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(),
+ f, new TestUtils.MockStateUpdater(), new Fields());
runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
Assert.assertArrayEquals(new Values[] { new Values(3), new Values(4)}, getCollectedValues().toArray());
}
@@ -84,7 +84,8 @@ public class TestPlanCompiler {
final AbstractTridentProcessor proc = compiler.compileForTest(state.tree());
final TridentTopology topo = proc.build(data);
Fields f = proc.outputStream().getOutputFields();
- proc.outputStream().each(f, new CollectDataFunction(), new Fields()).toStream();
+ proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(),
+ f, new TestUtils.MockStateUpdater(), new Fields());
runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
Assert.assertArrayEquals(new Values[] { new Values(0, 5L, 5, 1, 3, 4)}, getCollectedValues().toArray());
@@ -101,7 +102,8 @@ public class TestPlanCompiler {
final AbstractTridentProcessor proc = compiler.compileForTest(state.tree());
final TridentTopology topo = proc.build(data);
Fields f = proc.outputStream().getOutputFields();
- proc.outputStream().each(f, new CollectDataFunction(), new Fields()).toStream();
+ proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(),
+ f, new TestUtils.MockStateUpdater(), new Fields());
runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
Assert.assertArrayEquals(new Values[] { new Values(0, 5L, 39)}, getCollectedValues().toArray());
@@ -119,7 +121,8 @@ public class TestPlanCompiler {
final AbstractTridentProcessor proc = compiler.compileForTest(state.tree());
final TridentTopology topo = proc.build(data);
Fields f = proc.outputStream().getOutputFields();
- proc.outputStream().each(f, new CollectDataFunction(), new Fields()).toStream();
+ proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(),
+ f, new TestUtils.MockStateUpdater(), new Fields());
runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
assertListsAreEqualIgnoringOrder(Lists.newArrayList(new Values(1, 2L), new Values(0, 2L)), getCollectedValues());
@@ -137,7 +140,8 @@ public class TestPlanCompiler {
final AbstractTridentProcessor proc = compiler.compileForTest(state.tree());
final TridentTopology topo = proc.build(data);
Fields f = proc.outputStream().getOutputFields();
- proc.outputStream().each(f, new CollectDataFunction(), new Fields()).toStream();
+ proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(),
+ f, new TestUtils.MockStateUpdater(), new Fields());
runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
assertListsAreEqualIgnoringOrder(Lists.newArrayList(new Values(2, null), new Values(3, null), new Values(4, null)), getCollectedValues());
@@ -155,7 +159,8 @@ public class TestPlanCompiler {
final AbstractTridentProcessor proc = compiler.compileForTest(state.tree());
final TridentTopology topo = proc.build(data);
Fields f = proc.outputStream().getOutputFields();
- proc.outputStream().each(f, new CollectDataFunction(), new Fields()).toStream();
+ proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(),
+ f, new TestUtils.MockStateUpdater(), new Fields());
runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
assertListsAreEqualIgnoringOrder(Lists.newArrayList(new Values(2, null), new Values(3, null), new Values(4, null)), getCollectedValues());
@@ -173,7 +178,8 @@ public class TestPlanCompiler {
final AbstractTridentProcessor proc = compiler.compileForTest(state.tree());
final TridentTopology topo = proc.build(data);
Fields f = proc.outputStream().getOutputFields();
- proc.outputStream().each(f, new CollectDataFunction(), new Fields()).toStream();
+ proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(),
+ f, new TestUtils.MockStateUpdater(), new Fields());
runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
assertListsAreEqualIgnoringOrder(Lists.newArrayList(new Values(null, "dept-2"), new Values(null, "dept-3"), new Values(null, "dept-4"),
@@ -207,7 +213,8 @@ public class TestPlanCompiler {
AbstractTridentProcessor proc = compiler.compileForTest(state.tree());
final TridentTopology topo = proc.build(data);
Fields f = proc.outputStream().getOutputFields();
- proc.outputStream().each(f, new CollectDataFunction(), new Fields()).toStream();
+ proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(),
+ f, new TestUtils.MockStateUpdater(), new Fields());
runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
Assert.assertArrayEquals(new Values[] { new Values(true, false, true) }, getCollectedValues().toArray());
}
@@ -225,7 +232,8 @@ public class TestPlanCompiler {
AbstractTridentProcessor proc = compiler.compileForTest(state.tree());
final TridentTopology topo = proc.build(data);
Fields f = proc.outputStream().getOutputFields();
- proc.outputStream().each(f, new CollectDataFunction(), new Fields()).toStream();
+ proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(),
+ f, new TestUtils.MockStateUpdater(), new Fields());
runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
Assert.assertArrayEquals(new Values[] { new Values(5) }, getCollectedValues().toArray());
}
@@ -241,7 +249,8 @@ public class TestPlanCompiler {
AbstractTridentProcessor proc = compiler.compileForTest(state.tree());
final TridentTopology topo = proc.build(data);
Fields f = proc.outputStream().getOutputFields();
- proc.outputStream().each(f, new CollectDataFunction(), new Fields()).toStream();
+ proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(),
+ f, new TestUtils.MockStateUpdater(), new Fields());
runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
Assert.assertArrayEquals(new Values[] { new Values(0, 5L, 15L, 15L) }, getCollectedValues().toArray());
}
http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/JsonScheme.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/JsonScheme.java b/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/JsonScheme.java
deleted file mode 100644
index eed1282..0000000
--- a/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/JsonScheme.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.storm.sql.kafka;
-
-import org.apache.storm.spout.Scheme;
-import org.apache.storm.tuple.Fields;
-import org.apache.storm.utils.Utils;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-
-public class JsonScheme implements Scheme {
- private final List<String> fields;
-
- JsonScheme(List<String> fields) {
- this.fields = fields;
- }
-
- @Override
- public List<Object> deserialize(ByteBuffer ser) {
- ObjectMapper mapper = new ObjectMapper();
- try {
- @SuppressWarnings("unchecked")
- HashMap<String, Object> map = mapper.readValue(Utils.toByteArray(ser), HashMap.class);
- ArrayList<Object> list = new ArrayList<>();
- for (String f : fields) {
- list.add(map.get(f));
- }
- return list;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public Fields getOutputFields() {
- return new Fields(fields);
- }
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/JsonSerializer.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/JsonSerializer.java b/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/JsonSerializer.java
deleted file mode 100644
index e3d5d01..0000000
--- a/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/JsonSerializer.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.storm.sql.kafka;
-
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.google.common.base.Preconditions;
-import org.apache.storm.sql.runtime.IOutputSerializer;
-
-import java.io.IOException;
-import java.io.StringWriter;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
-class JsonSerializer implements IOutputSerializer {
- private final List<String> fieldNames;
- private transient final JsonFactory jsonFactory;
-
- JsonSerializer(List<String> fieldNames) {
- this.fieldNames = fieldNames;
- jsonFactory = new JsonFactory();
- }
-
- @Override
- public ByteBuffer write(List<Object> data, ByteBuffer buffer) {
- Preconditions.checkArgument(data != null && data.size() == fieldNames.size(), "Invalid schema");
- StringWriter sw = new StringWriter();
- try (JsonGenerator jg = jsonFactory.createGenerator(sw)) {
- jg.writeStartObject();
- for (int i = 0; i < fieldNames.size(); ++i) {
- jg.writeFieldName(fieldNames.get(i));
- jg.writeObject(data.get(i));
- }
- jg.writeEndObject();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return ByteBuffer.wrap(sw.toString().getBytes(StandardCharsets.UTF_8));
- }
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/KafkaDataSourcesProvider.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/KafkaDataSourcesProvider.java b/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/KafkaDataSourcesProvider.java
index 9f8eeac..19fab05 100644
--- a/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/KafkaDataSourcesProvider.java
+++ b/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/KafkaDataSourcesProvider.java
@@ -17,6 +17,8 @@
*/
package org.apache.storm.sql.kafka;
+import org.apache.storm.kafka.trident.TridentKafkaStateFactory;
+import org.apache.storm.kafka.trident.TridentKafkaUpdater;
import org.apache.storm.spout.SchemeAsMultiScheme;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
@@ -24,14 +26,10 @@ import org.apache.storm.sql.runtime.*;
import org.apache.storm.kafka.ZkHosts;
import org.apache.storm.kafka.trident.OpaqueTridentKafkaSpout;
import org.apache.storm.kafka.trident.TridentKafkaConfig;
-import org.apache.storm.kafka.trident.TridentKafkaState;
import org.apache.storm.kafka.trident.mapper.TridentTupleToKafkaMapper;
import org.apache.storm.kafka.trident.selector.DefaultTopicSelector;
-import org.apache.storm.kafka.trident.selector.KafkaTopicSelector;
-import org.apache.storm.trident.operation.BaseFunction;
-import org.apache.storm.trident.operation.Function;
-import org.apache.storm.trident.operation.TridentCollector;
-import org.apache.storm.trident.operation.TridentOperationContext;
+import org.apache.storm.sql.runtime.serde.json.JsonScheme;
+import org.apache.storm.sql.runtime.serde.json.JsonSerializer;
import org.apache.storm.trident.spout.ITridentDataSource;
import org.apache.storm.trident.tuple.TridentTuple;
@@ -68,42 +66,6 @@ public class KafkaDataSourcesProvider implements DataSourcesProvider {
}
}
- static class KafkaTridentSink extends BaseFunction {
- private transient TridentKafkaState state;
- private final String topic;
- private final int primaryKeyIndex;
- private final Properties producerProperties;
- private final List<String> fieldNames;
-
- private KafkaTridentSink(String topic, int primaryKeyIndex, Properties producerProperties,
- List<String> fieldNames) {
- this.topic = topic;
- this.primaryKeyIndex = primaryKeyIndex;
- this.producerProperties = producerProperties;
- this.fieldNames = fieldNames;
- }
-
- @Override
- public void cleanup() {
- super.cleanup();
- }
-
- @Override
- public void prepare(Map conf, TridentOperationContext context) {
- JsonSerializer serializer = new JsonSerializer(fieldNames);
- SqlKafkaMapper m = new SqlKafkaMapper(primaryKeyIndex, serializer);
- state = new TridentKafkaState()
- .withKafkaTopicSelector(new DefaultTopicSelector(topic))
- .withTridentTupleToKafkaMapper(m);
- state.prepare(producerProperties);
- }
-
- @Override
- public void execute(TridentTuple tuple, TridentCollector collector) {
- state.updateState(Collections.singletonList(tuple), collector);
- }
- }
-
private static class KafkaTridentDataSource implements ISqlTridentDataSource {
private final TridentKafkaConfig conf;
private final String topic;
@@ -125,7 +87,7 @@ public class KafkaDataSourcesProvider implements DataSourcesProvider {
}
@Override
- public Function getConsumer() {
+ public SqlTridentConsumer getConsumer() {
Preconditions.checkNotNull(producerProperties,
"Writable Kafka Table " + topic + " must contain producer config");
Properties props = new Properties();
@@ -141,7 +103,18 @@ public class KafkaDataSourcesProvider implements DataSourcesProvider {
}
Preconditions.checkState(props.containsKey("bootstrap.servers"),
"Writable Kafka Table " + topic + " must contain \"bootstrap.servers\" config");
- return new KafkaTridentSink(topic, primaryKeyIndex, props, fields);
+
+ JsonSerializer serializer = new JsonSerializer(fields);
+ SqlKafkaMapper mapper = new SqlKafkaMapper(primaryKeyIndex, serializer);
+
+ TridentKafkaStateFactory stateFactory = new TridentKafkaStateFactory()
+ .withKafkaTopicSelector(new DefaultTopicSelector(topic))
+ .withProducerProperties(props)
+ .withTridentTupleToKafkaMapper(mapper);
+
+ TridentKafkaUpdater stateUpdater = new TridentKafkaUpdater();
+
+ return new SimpleSqlTridentConsumer(stateFactory, stateUpdater);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestJsonRepresentation.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestJsonRepresentation.java b/external/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestJsonRepresentation.java
deleted file mode 100644
index 7e85410..0000000
--- a/external/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestJsonRepresentation.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.storm.sql.kafka;
-
-import org.apache.storm.utils.Utils;
-import com.google.common.collect.Lists;
-import org.junit.Test;
-
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.List;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-
-public class TestJsonRepresentation {
- @Test
- public void testJsonScheme() {
- final List<String> fields = Lists.newArrayList("ID", "val");
- final String s = "{\"ID\": 1, \"val\": \"2\"}";
- JsonScheme scheme = new JsonScheme(fields);
- List<Object> o = scheme.deserialize(ByteBuffer.wrap(s.getBytes(Charset.defaultCharset())));
- assertArrayEquals(new Object[] {1, "2"}, o.toArray());
- }
-
- @Test
- public void testJsonSerializer() {
- final List<String> fields = Lists.newArrayList("ID", "val");
- List<Object> o = Lists.<Object> newArrayList(1, "2");
- JsonSerializer s = new JsonSerializer(fields);
- ByteBuffer buf = s.write(o, null);
- byte[] b = Utils.toByteArray(buf);
- assertEquals("{\"ID\":1,\"val\":\"2\"}", new String(b));
- }
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestKafkaDataSourcesProvider.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestKafkaDataSourcesProvider.java b/external/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestKafkaDataSourcesProvider.java
index f6e75ab..2725893 100644
--- a/external/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestKafkaDataSourcesProvider.java
+++ b/external/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestKafkaDataSourcesProvider.java
@@ -22,16 +22,18 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.storm.sql.kafka.KafkaDataSourcesProvider.KafkaTridentSink;
+import org.apache.storm.kafka.trident.TridentKafkaState;
+import org.apache.storm.kafka.trident.TridentKafkaStateFactory;
+import org.apache.storm.kafka.trident.TridentKafkaUpdater;
import org.apache.storm.sql.runtime.DataSourcesRegistry;
import org.apache.storm.sql.runtime.FieldInfo;
import org.apache.storm.sql.runtime.ISqlTridentDataSource;
+import org.apache.storm.sql.runtime.serde.json.JsonSerializer;
+import org.apache.storm.trident.tuple.TridentTuple;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentMatcher;
import org.mockito.internal.util.reflection.Whitebox;
-import org.apache.storm.kafka.trident.TridentKafkaState;
-import org.apache.storm.trident.tuple.TridentTuple;
import java.net.URI;
import java.nio.ByteBuffer;
@@ -40,7 +42,12 @@ import java.util.Collections;
import java.util.List;
import java.util.concurrent.Future;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.argThat;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
public class TestKafkaDataSourcesProvider {
private static final List<FieldInfo> FIELDS = ImmutableList.of(
@@ -63,12 +70,17 @@ public class TestKafkaDataSourcesProvider {
ISqlTridentDataSource ds = DataSourcesRegistry.constructTridentDataSource(
URI.create("kafka://mock?topic=foo"), null, null, TBL_PROPERTIES, FIELDS);
Assert.assertNotNull(ds);
- KafkaTridentSink sink = (KafkaTridentSink) ds.getConsumer();
- sink.prepare(null, null);
- TridentKafkaState state = (TridentKafkaState) Whitebox.getInternalState(sink, "state");
+
+ ISqlTridentDataSource.SqlTridentConsumer consumer = ds.getConsumer();
+
+ Assert.assertEquals(TridentKafkaStateFactory.class, consumer.getStateFactory().getClass());
+ Assert.assertEquals(TridentKafkaUpdater.class, consumer.getStateUpdater().getClass());
+
+ TridentKafkaState state = (TridentKafkaState) consumer.getStateFactory().makeState(Collections.emptyMap(), null, 0, 1);
KafkaProducer producer = mock(KafkaProducer.class);
doReturn(mock(Future.class)).when(producer).send(any(ProducerRecord.class));
Whitebox.setInternalState(state, "producer", producer);
+
List<TridentTuple> tupleList = mockTupleList();
for (TridentTuple t : tupleList) {
state.updateState(Collections.singletonList(t), null);
http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-runtime/pom.xml b/external/sql/storm-sql-runtime/pom.xml
index 6663039..fd931f4 100644
--- a/external/sql/storm-sql-runtime/pom.xml
+++ b/external/sql/storm-sql-runtime/pom.xml
@@ -73,6 +73,10 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
<!-- janino -->
<dependency>
<groupId>org.codehaus.janino</groupId>
http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/FieldInfo.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/FieldInfo.java b/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/FieldInfo.java
index cb1176b..03b030b 100644
--- a/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/FieldInfo.java
+++ b/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/FieldInfo.java
@@ -17,10 +17,12 @@
*/
package org.apache.storm.sql.runtime;
+import java.io.Serializable;
+
/**
* Describe each column of the field
*/
-public class FieldInfo {
+public class FieldInfo implements Serializable {
private final String name;
private final Class<?> type;
private final boolean isPrimary;
http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/ISqlTridentDataSource.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/ISqlTridentDataSource.java b/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/ISqlTridentDataSource.java
index 92961dc..9eae5ae 100644
--- a/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/ISqlTridentDataSource.java
+++ b/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/ISqlTridentDataSource.java
@@ -17,14 +17,49 @@
*/
package org.apache.storm.sql.runtime;
-import org.apache.storm.trident.operation.Function;
-import org.apache.storm.trident.spout.IBatchSpout;
import org.apache.storm.trident.spout.ITridentDataSource;
+import org.apache.storm.trident.state.StateFactory;
+import org.apache.storm.trident.state.StateUpdater;
/**
* A ISqlTridentDataSource specifies how an external data source produces and consumes data.
*/
public interface ISqlTridentDataSource {
+ /**
+ * SqlTridentConsumer is a data structure containing StateFactory and StateUpdater for consuming tuples with State.
+ *
+ * Please note that StateFactory and StateUpdater should use same class which implements State.
+ *
+ * @see org.apache.storm.trident.state.StateFactory
+ * @see org.apache.storm.trident.state.StateUpdater
+ */
+ interface SqlTridentConsumer {
+ StateFactory getStateFactory();
+ StateUpdater getStateUpdater();
+ }
+
+ /**
+ * Provides instance of ITridentDataSource which can be used as producer in Trident.
+ *
+ * Since ITridentDataSource is a marker interface for Trident Spout interfaces, this method should effectively
+ * return an instance of one of these interfaces (can be changed if Trident API evolves) or descendants:
+ * - IBatchSpout
+ * - ITridentSpout
+ * - IPartitionedTridentSpout
+ * - IOpaquePartitionedTridentSpout
+ *
+ * @see org.apache.storm.trident.spout.ITridentDataSource
+ * @see org.apache.storm.trident.spout.IBatchSpout
+ * @see org.apache.storm.trident.spout.ITridentSpout
+ * @see org.apache.storm.trident.spout.IPartitionedTridentSpout
+ * @see org.apache.storm.trident.spout.IOpaquePartitionedTridentSpout
+ */
ITridentDataSource getProducer();
- Function getConsumer();
+
+ /**
+ * Provides instance of SqlTridentConsumer which can be used as consumer (State) in Trident.
+ *
+ * @see SqlTridentConsumer
+ */
+ SqlTridentConsumer getConsumer();
}
http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/SimpleSqlTridentConsumer.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/SimpleSqlTridentConsumer.java b/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/SimpleSqlTridentConsumer.java
new file mode 100644
index 0000000..69d67d7
--- /dev/null
+++ b/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/SimpleSqlTridentConsumer.java
@@ -0,0 +1,24 @@
+package org.apache.storm.sql.runtime;
+
+import org.apache.storm.trident.state.StateFactory;
+import org.apache.storm.trident.state.StateUpdater;
+
+public class SimpleSqlTridentConsumer implements ISqlTridentDataSource.SqlTridentConsumer {
+ private final StateFactory stateFactory;
+ private final StateUpdater stateUpdater;
+
+ public SimpleSqlTridentConsumer(StateFactory stateFactory, StateUpdater stateUpdater) {
+ this.stateFactory = stateFactory;
+ this.stateUpdater = stateUpdater;
+ }
+
+ @Override
+ public StateFactory getStateFactory() {
+ return stateFactory;
+ }
+
+ @Override
+ public StateUpdater getStateUpdater() {
+ return stateUpdater;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonScheme.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonScheme.java b/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonScheme.java
new file mode 100644
index 0000000..3d4de0d
--- /dev/null
+++ b/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonScheme.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.sql.runtime.serde.json;
+
+import org.apache.storm.spout.Scheme;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.utils.Utils;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+public class JsonScheme implements Scheme {
+ private final List<String> fields;
+
+ public JsonScheme(List<String> fields) {
+ this.fields = fields;
+ }
+
+ @Override
+ public List<Object> deserialize(ByteBuffer ser) {
+ ObjectMapper mapper = new ObjectMapper();
+ try {
+ @SuppressWarnings("unchecked")
+ HashMap<String, Object> map = mapper.readValue(Utils.toByteArray(ser), HashMap.class);
+ ArrayList<Object> list = new ArrayList<>();
+ for (String f : fields) {
+ list.add(map.get(f));
+ }
+ return list;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Fields getOutputFields() {
+ return new Fields(fields);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonSerializer.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonSerializer.java b/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonSerializer.java
new file mode 100644
index 0000000..1e825c4
--- /dev/null
+++ b/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonSerializer.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.sql.runtime.serde.json;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.google.common.base.Preconditions;
+import org.apache.storm.sql.runtime.IOutputSerializer;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.io.StringWriter;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+public class JsonSerializer implements IOutputSerializer, Serializable {
+ private final List<String> fieldNames;
+ private final JsonFactory jsonFactory;
+
+ public JsonSerializer(List<String> fieldNames) {
+ this.fieldNames = fieldNames;
+ jsonFactory = new JsonFactory();
+ }
+
+ @Override
+ public ByteBuffer write(List<Object> data, ByteBuffer buffer) {
+ Preconditions.checkArgument(data != null && data.size() == fieldNames.size(), "Invalid schema");
+ StringWriter sw = new StringWriter();
+ try (JsonGenerator jg = jsonFactory.createGenerator(sw)) {
+ jg.writeStartObject();
+ for (int i = 0; i < fieldNames.size(); ++i) {
+ jg.writeFieldName(fieldNames.get(i));
+ jg.writeObject(data.get(i));
+ }
+ jg.writeEndObject();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return ByteBuffer.wrap(sw.toString().getBytes(StandardCharsets.UTF_8));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestJsonRepresentation.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestJsonRepresentation.java b/external/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestJsonRepresentation.java
new file mode 100644
index 0000000..6ca1877
--- /dev/null
+++ b/external/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestJsonRepresentation.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.sql;
+
+import org.apache.storm.sql.runtime.serde.json.JsonScheme;
+import org.apache.storm.sql.runtime.serde.json.JsonSerializer;
+import org.apache.storm.utils.Utils;
+import com.google.common.collect.Lists;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.List;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class TestJsonRepresentation {
+ @Test
+ public void testJsonScheme() {
+ final List<String> fields = Lists.newArrayList("ID", "val");
+ final String s = "{\"ID\": 1, \"val\": \"2\"}";
+ JsonScheme scheme = new JsonScheme(fields);
+ List<Object> o = scheme.deserialize(ByteBuffer.wrap(s.getBytes(Charset.defaultCharset())));
+ assertArrayEquals(new Object[] {1, "2"}, o.toArray());
+ }
+
+ @Test
+ public void testJsonSerializer() {
+ final List<String> fields = Lists.newArrayList("ID", "val");
+ List<Object> o = Lists.<Object> newArrayList(1, "2");
+ JsonSerializer s = new JsonSerializer(fields);
+ ByteBuffer buf = s.write(o, null);
+ byte[] b = Utils.toByteArray(buf);
+ assertEquals("{\"ID\":1,\"val\":\"2\"}", new String(b));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestUtils.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestUtils.java b/external/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestUtils.java
index 58c3f26..7648f58 100644
--- a/external/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestUtils.java
+++ b/external/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestUtils.java
@@ -21,7 +21,13 @@ package org.apache.storm.sql;
import org.apache.storm.ILocalCluster;
import org.apache.storm.LocalCluster;
+import org.apache.storm.sql.runtime.SimpleSqlTridentConsumer;
+import org.apache.storm.task.IMetricsContext;
import org.apache.storm.task.TopologyContext;
+import org.apache.storm.trident.operation.TridentOperationContext;
+import org.apache.storm.trident.state.State;
+import org.apache.storm.trident.state.StateFactory;
+import org.apache.storm.trident.state.StateUpdater;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values;
import org.apache.storm.sql.runtime.ChannelContext;
@@ -188,31 +194,70 @@ public class TestUtils {
}
}
- public static class MockSqlTridentDataSource implements ISqlTridentDataSource {
- @Override
- public IBatchSpout getProducer() {
- return new MockSpout();
+ public static class MockState implements State {
+ /**
+ * Collect all values in a static variable as the instance will go through serialization and deserialization.
+ * NOTE: This should be cleared before or after running each test.
+ */
+ private transient static final List<List<Object> > VALUES = new ArrayList<>();
+
+ public static List<List<Object>> getCollectedValues() {
+ return VALUES;
}
@Override
- public Function getConsumer() {
- return new CollectDataFunction();
+ public void beginCommit(Long txid) {
+ // NOOP
}
- public static class CollectDataFunction extends BaseFunction {
- /**
- * Collect all values in a static variable as the instance will go through serialization and deserialization.
- */
- private transient static final List<List<Object> > VALUES = new ArrayList<>();
- public static List<List<Object>> getCollectedValues() {
- return VALUES;
- }
+ @Override
+ public void commit(Long txid) {
+ // NOOP
+ }
- @Override
- public void execute(TridentTuple tuple, TridentCollector collector) {
+ public void updateState(List<TridentTuple> tuples, TridentCollector collector) {
+ for (TridentTuple tuple : tuples) {
VALUES.add(tuple.getValues());
}
}
+ }
+
+ public static class MockStateFactory implements StateFactory {
+
+ @Override
+ public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
+ return new MockState();
+ }
+ }
+
+ public static class MockStateUpdater implements StateUpdater<MockState> {
+
+ @Override
+ public void updateState(MockState state, List<TridentTuple> tuples, TridentCollector collector) {
+ state.updateState(tuples, collector);
+ }
+
+ @Override
+ public void prepare(Map conf, TridentOperationContext context) {
+ // NOOP
+ }
+
+ @Override
+ public void cleanup() {
+ // NOOP
+ }
+ }
+
+ public static class MockSqlTridentDataSource implements ISqlTridentDataSource {
+ @Override
+ public IBatchSpout getProducer() {
+ return new MockSpout();
+ }
+
+ @Override
+ public SqlTridentConsumer getConsumer() {
+ return new SimpleSqlTridentConsumer(new MockStateFactory(), new MockStateUpdater());
+ }
private static class MockSpout implements IBatchSpout {
private final ArrayList<Values> RECORDS = new ArrayList<>();
@@ -269,23 +314,8 @@ public class TestUtils {
}
@Override
- public Function getConsumer() {
- return new CollectDataFunction();
- }
-
- public static class CollectDataFunction extends BaseFunction {
- /**
- * Collect all values in a static variable as the instance will go through serialization and deserialization.
- */
- private transient static final List<List<Object> > VALUES = new ArrayList<>();
- public static List<List<Object>> getCollectedValues() {
- return VALUES;
- }
-
- @Override
- public void execute(TridentTuple tuple, TridentCollector collector) {
- VALUES.add(tuple.getValues());
- }
+ public SqlTridentConsumer getConsumer() {
+ return new SimpleSqlTridentConsumer(new MockStateFactory(), new MockStateUpdater());
}
private static class MockGroupedSpout implements IBatchSpout {
@@ -343,23 +373,8 @@ public class TestUtils {
}
@Override
- public Function getConsumer() {
- return new CollectDataFunction();
- }
-
- public static class CollectDataFunction extends BaseFunction {
- /**
- * Collect all values in a static variable as the instance will go through serialization and deserialization.
- */
- private transient static final List<List<Object> > VALUES = new ArrayList<>();
- public static List<List<Object>> getCollectedValues() {
- return VALUES;
- }
-
- @Override
- public void execute(TridentTuple tuple, TridentCollector collector) {
- VALUES.add(tuple.getValues());
- }
+ public SqlTridentConsumer getConsumer() {
+ return new SimpleSqlTridentConsumer(new MockStateFactory(), new MockStateUpdater());
}
private static class MockSpout implements IBatchSpout {
@@ -420,23 +435,8 @@ public class TestUtils {
}
@Override
- public Function getConsumer() {
- return new CollectDataFunction();
- }
-
- public static class CollectDataFunction extends BaseFunction {
- /**
- * Collect all values in a static variable as the instance will go through serialization and deserialization.
- */
- private transient static final List<List<Object> > VALUES = new ArrayList<>();
- public static List<List<Object>> getCollectedValues() {
- return VALUES;
- }
-
- @Override
- public void execute(TridentTuple tuple, TridentCollector collector) {
- VALUES.add(tuple.getValues());
- }
+ public SqlTridentConsumer getConsumer() {
+ return new SimpleSqlTridentConsumer(new MockStateFactory(), new MockStateUpdater());
}
private static class MockSpout implements IBatchSpout {
@@ -518,8 +518,4 @@ public class TestUtils {
final long NANOSECONDS_PER_MILLISECOND = 1000000;
return System.nanoTime() / NANOSECONDS_PER_MILLISECOND;
}
-
- public static ILocalCluster newLocalCluster() {
- return new LocalCluster();
- }
}
[3/3] storm git commit: add STORM-2089 to CHANGELOG
Posted by ka...@apache.org.
add STORM-2089 to CHANGELOG
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8fbef379
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8fbef379
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8fbef379
Branch: refs/heads/1.x-branch
Commit: 8fbef3793cf1d18e90ec8e756ce79b1fea3ca616
Parents: c4a69c2
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Sep 27 06:52:34 2016 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Sep 27 06:52:34 2016 +0900
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/8fbef379/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2bcfe65..e0a40b3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 1.1.0
+ * STORM-2089: Replace Consumer of ISqlTridentDataSource with SqlTridentConsumer
* STORM-2118: A few fixes for storm-sql standalone mode
* STORM-2105: Cluster/Supervisor total and available resources displayed in the UI
* STORM-2078: enable paging in worker datatable
[2/3] storm git commit: Merge branch 'STORM-2089-1.x' into 1.x-branch
Posted by ka...@apache.org.
Merge branch 'STORM-2089-1.x' into 1.x-branch
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/c4a69c24
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/c4a69c24
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/c4a69c24
Branch: refs/heads/1.x-branch
Commit: c4a69c24a47f236bffc8086b57aeb78507067d75
Parents: 8bcdd7b 130b051
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Sep 27 06:51:48 2016 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Sep 27 06:51:48 2016 +0900
----------------------------------------------------------------------
.../trident/TridentLogicalPlanCompiler.java | 9 +-
.../backends/trident/TestPlanCompiler.java | 33 +++--
.../org/apache/storm/sql/kafka/JsonScheme.java | 58 --------
.../apache/storm/sql/kafka/JsonSerializer.java | 56 --------
.../sql/kafka/KafkaDataSourcesProvider.java | 61 +++-----
.../storm/sql/kafka/TestJsonRepresentation.java | 50 -------
.../sql/kafka/TestKafkaDataSourcesProvider.java | 26 +++-
external/sql/storm-sql-runtime/pom.xml | 4 +
.../org/apache/storm/sql/runtime/FieldInfo.java | 4 +-
.../sql/runtime/ISqlTridentDataSource.java | 41 +++++-
.../sql/runtime/SimpleSqlTridentConsumer.java | 24 ++++
.../sql/runtime/serde/json/JsonScheme.java | 58 ++++++++
.../sql/runtime/serde/json/JsonSerializer.java | 57 ++++++++
.../storm/sql/TestJsonRepresentation.java | 52 +++++++
.../test/org/apache/storm/sql/TestUtils.java | 138 +++++++++----------
15 files changed, 367 insertions(+), 304 deletions(-)
----------------------------------------------------------------------