You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2016/09/26 21:52:46 UTC

[1/3] storm git commit: STORM-2089 Replace Consumer of ISqlTridentDataSource with SqlTridentConsumer

Repository: storm
Updated Branches:
  refs/heads/1.x-branch 8bcdd7b5c -> 8fbef3793


STORM-2089 Replace Consumer of ISqlTridentDataSource with SqlTridentConsumer

* SqlTridentConsumer contains StateFactory and StateUpdater which is needed to store tuples to State via batch
* Apply the change to storm-sql-kafka
* move out JsonScheme and JsonSerializer to runtime
  * it will be used from other external sql modules
* add javadoc to ISqlTridentDataSource


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/130b0514
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/130b0514
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/130b0514

Branch: refs/heads/1.x-branch
Commit: 130b0514dd8dd6ec382bc274da98f317dbb61315
Parents: 8bcdd7b
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Sep 13 11:40:13 2016 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Sep 27 06:50:25 2016 +0900

----------------------------------------------------------------------
 .../trident/TridentLogicalPlanCompiler.java     |   9 +-
 .../backends/trident/TestPlanCompiler.java      |  33 +++--
 .../org/apache/storm/sql/kafka/JsonScheme.java  |  58 --------
 .../apache/storm/sql/kafka/JsonSerializer.java  |  56 --------
 .../sql/kafka/KafkaDataSourcesProvider.java     |  61 +++-----
 .../storm/sql/kafka/TestJsonRepresentation.java |  50 -------
 .../sql/kafka/TestKafkaDataSourcesProvider.java |  26 +++-
 external/sql/storm-sql-runtime/pom.xml          |   4 +
 .../org/apache/storm/sql/runtime/FieldInfo.java |   4 +-
 .../sql/runtime/ISqlTridentDataSource.java      |  41 +++++-
 .../sql/runtime/SimpleSqlTridentConsumer.java   |  24 ++++
 .../sql/runtime/serde/json/JsonScheme.java      |  58 ++++++++
 .../sql/runtime/serde/json/JsonSerializer.java  |  57 ++++++++
 .../storm/sql/TestJsonRepresentation.java       |  52 +++++++
 .../test/org/apache/storm/sql/TestUtils.java    | 138 +++++++++----------
 15 files changed, 367 insertions(+), 304 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-core/src/jvm/org/apache/storm/sql/compiler/backends/trident/TridentLogicalPlanCompiler.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-core/src/jvm/org/apache/storm/sql/compiler/backends/trident/TridentLogicalPlanCompiler.java b/external/sql/storm-sql-core/src/jvm/org/apache/storm/sql/compiler/backends/trident/TridentLogicalPlanCompiler.java
index 688b164..a0bfa21 100644
--- a/external/sql/storm-sql-core/src/jvm/org/apache/storm/sql/compiler/backends/trident/TridentLogicalPlanCompiler.java
+++ b/external/sql/storm-sql-core/src/jvm/org/apache/storm/sql/compiler/backends/trident/TridentLogicalPlanCompiler.java
@@ -131,8 +131,13 @@ public class TridentLogicalPlanCompiler extends PostOrderRelNodeVisitor<IAggrega
         List<String> inputFields = input.getRowType().getFieldNames();
         List<String> outputFields = modify.getRowType().getFieldNames();
 
-        return inputStream.each(new Fields(inputFields), sources.get(tableName).getConsumer(), new Fields(outputFields))
-                .name(stageName);
+        ISqlTridentDataSource.SqlTridentConsumer consumer = sources.get(tableName).getConsumer();
+
+        // In fact this is normally the end of stream, but to match the return type we open new streams based on State values
+        return inputStream
+                .partitionPersist(consumer.getStateFactory(), new Fields(inputFields), consumer.getStateUpdater(),
+                        new Fields(outputFields))
+                .newValuesStream().name(stageName);
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/trident/TestPlanCompiler.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/trident/TestPlanCompiler.java b/external/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/trident/TestPlanCompiler.java
index 35d1364..85510a0 100644
--- a/external/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/trident/TestPlanCompiler.java
+++ b/external/sql/storm-sql-core/src/test/org/apache/storm/sql/compiler/backends/trident/TestPlanCompiler.java
@@ -29,7 +29,6 @@ import org.apache.storm.ILocalCluster;
 import org.apache.storm.LocalCluster;
 import org.apache.storm.generated.StormTopology;
 import org.apache.storm.sql.TestUtils;
-import org.apache.storm.sql.TestUtils.MockSqlTridentDataSource.CollectDataFunction;
 import org.apache.storm.sql.compiler.TestCompilerUtils;
 import org.apache.storm.sql.runtime.ISqlTridentDataSource;
 import org.apache.storm.sql.runtime.trident.AbstractTridentProcessor;
@@ -46,7 +45,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Callable;
 
-import static org.apache.storm.sql.TestUtils.MockSqlTridentDataSource.CollectDataFunction.getCollectedValues;
+import static org.apache.storm.sql.TestUtils.MockState.getCollectedValues;
 
 public class TestPlanCompiler {
   private final JavaTypeFactory typeFactory = new JavaTypeFactoryImpl(
@@ -68,7 +67,8 @@ public class TestPlanCompiler {
     final AbstractTridentProcessor proc = compiler.compileForTest(state.tree());
     final TridentTopology topo = proc.build(data);
     Fields f = proc.outputStream().getOutputFields();
-    proc.outputStream().each(f, new CollectDataFunction(), new Fields()).toStream();
+    proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(),
+            f, new TestUtils.MockStateUpdater(), new Fields());
     runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
     Assert.assertArrayEquals(new Values[] { new Values(3), new Values(4)}, getCollectedValues().toArray());
   }
@@ -84,7 +84,8 @@ public class TestPlanCompiler {
     final AbstractTridentProcessor proc = compiler.compileForTest(state.tree());
     final TridentTopology topo = proc.build(data);
     Fields f = proc.outputStream().getOutputFields();
-    proc.outputStream().each(f, new CollectDataFunction(), new Fields()).toStream();
+    proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(),
+            f, new TestUtils.MockStateUpdater(), new Fields());
     runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
 
     Assert.assertArrayEquals(new Values[] { new Values(0, 5L, 5, 1, 3, 4)}, getCollectedValues().toArray());
@@ -101,7 +102,8 @@ public class TestPlanCompiler {
     final AbstractTridentProcessor proc = compiler.compileForTest(state.tree());
     final TridentTopology topo = proc.build(data);
     Fields f = proc.outputStream().getOutputFields();
-    proc.outputStream().each(f, new CollectDataFunction(), new Fields()).toStream();
+    proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(),
+            f, new TestUtils.MockStateUpdater(), new Fields());
     runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
 
     Assert.assertArrayEquals(new Values[] { new Values(0, 5L, 39)}, getCollectedValues().toArray());
@@ -119,7 +121,8 @@ public class TestPlanCompiler {
     final AbstractTridentProcessor proc = compiler.compileForTest(state.tree());
     final TridentTopology topo = proc.build(data);
     Fields f = proc.outputStream().getOutputFields();
-    proc.outputStream().each(f, new CollectDataFunction(), new Fields()).toStream();
+    proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(),
+            f, new TestUtils.MockStateUpdater(), new Fields());
     runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
 
     assertListsAreEqualIgnoringOrder(Lists.newArrayList(new Values(1, 2L), new Values(0, 2L)), getCollectedValues());
@@ -137,7 +140,8 @@ public class TestPlanCompiler {
     final AbstractTridentProcessor proc = compiler.compileForTest(state.tree());
     final TridentTopology topo = proc.build(data);
     Fields f = proc.outputStream().getOutputFields();
-    proc.outputStream().each(f, new CollectDataFunction(), new Fields()).toStream();
+    proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(),
+            f, new TestUtils.MockStateUpdater(), new Fields());
     runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
 
     assertListsAreEqualIgnoringOrder(Lists.newArrayList(new Values(2, null), new Values(3, null), new Values(4, null)), getCollectedValues());
@@ -155,7 +159,8 @@ public class TestPlanCompiler {
     final AbstractTridentProcessor proc = compiler.compileForTest(state.tree());
     final TridentTopology topo = proc.build(data);
     Fields f = proc.outputStream().getOutputFields();
-    proc.outputStream().each(f, new CollectDataFunction(), new Fields()).toStream();
+    proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(),
+            f, new TestUtils.MockStateUpdater(), new Fields());
     runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
 
     assertListsAreEqualIgnoringOrder(Lists.newArrayList(new Values(2, null), new Values(3, null), new Values(4, null)), getCollectedValues());
@@ -173,7 +178,8 @@ public class TestPlanCompiler {
     final AbstractTridentProcessor proc = compiler.compileForTest(state.tree());
     final TridentTopology topo = proc.build(data);
     Fields f = proc.outputStream().getOutputFields();
-    proc.outputStream().each(f, new CollectDataFunction(), new Fields()).toStream();
+    proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(),
+            f, new TestUtils.MockStateUpdater(), new Fields());
     runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
 
     assertListsAreEqualIgnoringOrder(Lists.newArrayList(new Values(null, "dept-2"), new Values(null, "dept-3"), new Values(null, "dept-4"),
@@ -207,7 +213,8 @@ public class TestPlanCompiler {
     AbstractTridentProcessor proc = compiler.compileForTest(state.tree());
     final TridentTopology topo = proc.build(data);
     Fields f = proc.outputStream().getOutputFields();
-    proc.outputStream().each(f, new CollectDataFunction(), new Fields()).toStream();
+    proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(),
+            f, new TestUtils.MockStateUpdater(), new Fields());
     runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
     Assert.assertArrayEquals(new Values[] { new Values(true, false, true) }, getCollectedValues().toArray());
   }
@@ -225,7 +232,8 @@ public class TestPlanCompiler {
     AbstractTridentProcessor proc = compiler.compileForTest(state.tree());
     final TridentTopology topo = proc.build(data);
     Fields f = proc.outputStream().getOutputFields();
-    proc.outputStream().each(f, new CollectDataFunction(), new Fields()).toStream();
+    proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(),
+            f, new TestUtils.MockStateUpdater(), new Fields());
     runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
     Assert.assertArrayEquals(new Values[] { new Values(5) }, getCollectedValues().toArray());
   }
@@ -241,7 +249,8 @@ public class TestPlanCompiler {
     AbstractTridentProcessor proc = compiler.compileForTest(state.tree());
     final TridentTopology topo = proc.build(data);
     Fields f = proc.outputStream().getOutputFields();
-    proc.outputStream().each(f, new CollectDataFunction(), new Fields()).toStream();
+    proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(),
+            f, new TestUtils.MockStateUpdater(), new Fields());
     runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
     Assert.assertArrayEquals(new Values[] { new Values(0, 5L, 15L, 15L) }, getCollectedValues().toArray());
   }

http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/JsonScheme.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/JsonScheme.java b/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/JsonScheme.java
deleted file mode 100644
index eed1282..0000000
--- a/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/JsonScheme.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.storm.sql.kafka;
-
-import org.apache.storm.spout.Scheme;
-import org.apache.storm.tuple.Fields;
-import org.apache.storm.utils.Utils;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-
-public class JsonScheme implements Scheme {
-  private final List<String> fields;
-
-  JsonScheme(List<String> fields) {
-    this.fields = fields;
-  }
-
-  @Override
-  public List<Object> deserialize(ByteBuffer ser) {
-    ObjectMapper mapper = new ObjectMapper();
-    try {
-      @SuppressWarnings("unchecked")
-      HashMap<String, Object> map = mapper.readValue(Utils.toByteArray(ser), HashMap.class);
-      ArrayList<Object> list = new ArrayList<>();
-      for (String f : fields) {
-        list.add(map.get(f));
-      }
-      return list;
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  @Override
-  public Fields getOutputFields() {
-    return new Fields(fields);
-  }
-}

http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/JsonSerializer.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/JsonSerializer.java b/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/JsonSerializer.java
deleted file mode 100644
index e3d5d01..0000000
--- a/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/JsonSerializer.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.storm.sql.kafka;
-
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.google.common.base.Preconditions;
-import org.apache.storm.sql.runtime.IOutputSerializer;
-
-import java.io.IOException;
-import java.io.StringWriter;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
-class JsonSerializer implements IOutputSerializer {
-  private final List<String> fieldNames;
-  private transient final JsonFactory jsonFactory;
-
-  JsonSerializer(List<String> fieldNames) {
-    this.fieldNames = fieldNames;
-    jsonFactory = new JsonFactory();
-  }
-
-  @Override
-  public ByteBuffer write(List<Object> data, ByteBuffer buffer) {
-    Preconditions.checkArgument(data != null && data.size() == fieldNames.size(), "Invalid schema");
-    StringWriter sw = new StringWriter();
-    try (JsonGenerator jg = jsonFactory.createGenerator(sw)) {
-      jg.writeStartObject();
-      for (int i = 0; i < fieldNames.size(); ++i) {
-        jg.writeFieldName(fieldNames.get(i));
-        jg.writeObject(data.get(i));
-      }
-      jg.writeEndObject();
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
-    return ByteBuffer.wrap(sw.toString().getBytes(StandardCharsets.UTF_8));
-  }
-}

http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/KafkaDataSourcesProvider.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/KafkaDataSourcesProvider.java b/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/KafkaDataSourcesProvider.java
index 9f8eeac..19fab05 100644
--- a/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/KafkaDataSourcesProvider.java
+++ b/external/sql/storm-sql-external/storm-sql-kafka/src/jvm/org/apache/storm/sql/kafka/KafkaDataSourcesProvider.java
@@ -17,6 +17,8 @@
  */
 package org.apache.storm.sql.kafka;
 
+import org.apache.storm.kafka.trident.TridentKafkaStateFactory;
+import org.apache.storm.kafka.trident.TridentKafkaUpdater;
 import org.apache.storm.spout.SchemeAsMultiScheme;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Preconditions;
@@ -24,14 +26,10 @@ import org.apache.storm.sql.runtime.*;
 import org.apache.storm.kafka.ZkHosts;
 import org.apache.storm.kafka.trident.OpaqueTridentKafkaSpout;
 import org.apache.storm.kafka.trident.TridentKafkaConfig;
-import org.apache.storm.kafka.trident.TridentKafkaState;
 import org.apache.storm.kafka.trident.mapper.TridentTupleToKafkaMapper;
 import org.apache.storm.kafka.trident.selector.DefaultTopicSelector;
-import org.apache.storm.kafka.trident.selector.KafkaTopicSelector;
-import org.apache.storm.trident.operation.BaseFunction;
-import org.apache.storm.trident.operation.Function;
-import org.apache.storm.trident.operation.TridentCollector;
-import org.apache.storm.trident.operation.TridentOperationContext;
+import org.apache.storm.sql.runtime.serde.json.JsonScheme;
+import org.apache.storm.sql.runtime.serde.json.JsonSerializer;
 import org.apache.storm.trident.spout.ITridentDataSource;
 import org.apache.storm.trident.tuple.TridentTuple;
 
@@ -68,42 +66,6 @@ public class KafkaDataSourcesProvider implements DataSourcesProvider {
     }
   }
 
-  static class KafkaTridentSink extends BaseFunction {
-    private transient TridentKafkaState state;
-    private final String topic;
-    private final int primaryKeyIndex;
-    private final Properties producerProperties;
-    private final List<String> fieldNames;
-
-    private KafkaTridentSink(String topic, int primaryKeyIndex, Properties producerProperties,
-                             List<String> fieldNames) {
-      this.topic = topic;
-      this.primaryKeyIndex = primaryKeyIndex;
-      this.producerProperties = producerProperties;
-      this.fieldNames = fieldNames;
-    }
-
-    @Override
-    public void cleanup() {
-      super.cleanup();
-    }
-
-    @Override
-    public void prepare(Map conf, TridentOperationContext context) {
-      JsonSerializer serializer = new JsonSerializer(fieldNames);
-      SqlKafkaMapper m = new SqlKafkaMapper(primaryKeyIndex, serializer);
-      state = new TridentKafkaState()
-          .withKafkaTopicSelector(new DefaultTopicSelector(topic))
-          .withTridentTupleToKafkaMapper(m);
-      state.prepare(producerProperties);
-    }
-
-    @Override
-    public void execute(TridentTuple tuple, TridentCollector collector) {
-      state.updateState(Collections.singletonList(tuple), collector);
-    }
-  }
-
   private static class KafkaTridentDataSource implements ISqlTridentDataSource {
     private final TridentKafkaConfig conf;
     private final String topic;
@@ -125,7 +87,7 @@ public class KafkaDataSourcesProvider implements DataSourcesProvider {
     }
 
     @Override
-    public Function getConsumer() {
+    public SqlTridentConsumer getConsumer() {
       Preconditions.checkNotNull(producerProperties,
           "Writable Kafka Table " + topic + " must contain producer config");
       Properties props = new Properties();
@@ -141,7 +103,18 @@ public class KafkaDataSourcesProvider implements DataSourcesProvider {
       }
       Preconditions.checkState(props.containsKey("bootstrap.servers"),
           "Writable Kafka Table " + topic + " must contain \"bootstrap.servers\" config");
-      return new KafkaTridentSink(topic, primaryKeyIndex, props, fields);
+
+      JsonSerializer serializer = new JsonSerializer(fields);
+      SqlKafkaMapper mapper = new SqlKafkaMapper(primaryKeyIndex, serializer);
+
+      TridentKafkaStateFactory stateFactory = new TridentKafkaStateFactory()
+              .withKafkaTopicSelector(new DefaultTopicSelector(topic))
+              .withProducerProperties(props)
+              .withTridentTupleToKafkaMapper(mapper);
+
+      TridentKafkaUpdater stateUpdater = new TridentKafkaUpdater();
+
+      return new SimpleSqlTridentConsumer(stateFactory, stateUpdater);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestJsonRepresentation.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestJsonRepresentation.java b/external/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestJsonRepresentation.java
deleted file mode 100644
index 7e85410..0000000
--- a/external/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestJsonRepresentation.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.storm.sql.kafka;
-
-import org.apache.storm.utils.Utils;
-import com.google.common.collect.Lists;
-import org.junit.Test;
-
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.List;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-
-public class TestJsonRepresentation {
-  @Test
-  public void testJsonScheme() {
-    final List<String> fields = Lists.newArrayList("ID", "val");
-    final String s = "{\"ID\": 1, \"val\": \"2\"}";
-    JsonScheme scheme = new JsonScheme(fields);
-    List<Object> o = scheme.deserialize(ByteBuffer.wrap(s.getBytes(Charset.defaultCharset())));
-    assertArrayEquals(new Object[] {1, "2"}, o.toArray());
-  }
-
-  @Test
-  public void testJsonSerializer() {
-    final List<String> fields = Lists.newArrayList("ID", "val");
-    List<Object> o = Lists.<Object> newArrayList(1, "2");
-    JsonSerializer s = new JsonSerializer(fields);
-    ByteBuffer buf = s.write(o, null);
-    byte[] b = Utils.toByteArray(buf);
-    assertEquals("{\"ID\":1,\"val\":\"2\"}", new String(b));
-  }
-}

http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestKafkaDataSourcesProvider.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestKafkaDataSourcesProvider.java b/external/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestKafkaDataSourcesProvider.java
index f6e75ab..2725893 100644
--- a/external/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestKafkaDataSourcesProvider.java
+++ b/external/sql/storm-sql-external/storm-sql-kafka/src/test/org/apache/storm/sql/kafka/TestKafkaDataSourcesProvider.java
@@ -22,16 +22,18 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import org.apache.kafka.clients.producer.KafkaProducer;
 import org.apache.kafka.clients.producer.ProducerRecord;
-import org.apache.storm.sql.kafka.KafkaDataSourcesProvider.KafkaTridentSink;
+import org.apache.storm.kafka.trident.TridentKafkaState;
+import org.apache.storm.kafka.trident.TridentKafkaStateFactory;
+import org.apache.storm.kafka.trident.TridentKafkaUpdater;
 import org.apache.storm.sql.runtime.DataSourcesRegistry;
 import org.apache.storm.sql.runtime.FieldInfo;
 import org.apache.storm.sql.runtime.ISqlTridentDataSource;
+import org.apache.storm.sql.runtime.serde.json.JsonSerializer;
+import org.apache.storm.trident.tuple.TridentTuple;
 import org.junit.Assert;
 import org.junit.Test;
 import org.mockito.ArgumentMatcher;
 import org.mockito.internal.util.reflection.Whitebox;
-import org.apache.storm.kafka.trident.TridentKafkaState;
-import org.apache.storm.trident.tuple.TridentTuple;
 
 import java.net.URI;
 import java.nio.ByteBuffer;
@@ -40,7 +42,12 @@ import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.Future;
 
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.argThat;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
 
 public class TestKafkaDataSourcesProvider {
   private static final List<FieldInfo> FIELDS = ImmutableList.of(
@@ -63,12 +70,17 @@ public class TestKafkaDataSourcesProvider {
     ISqlTridentDataSource ds = DataSourcesRegistry.constructTridentDataSource(
         URI.create("kafka://mock?topic=foo"), null, null, TBL_PROPERTIES, FIELDS);
     Assert.assertNotNull(ds);
-    KafkaTridentSink sink = (KafkaTridentSink) ds.getConsumer();
-    sink.prepare(null, null);
-    TridentKafkaState state = (TridentKafkaState) Whitebox.getInternalState(sink, "state");
+
+    ISqlTridentDataSource.SqlTridentConsumer consumer = ds.getConsumer();
+
+    Assert.assertEquals(TridentKafkaStateFactory.class, consumer.getStateFactory().getClass());
+    Assert.assertEquals(TridentKafkaUpdater.class, consumer.getStateUpdater().getClass());
+
+    TridentKafkaState state = (TridentKafkaState) consumer.getStateFactory().makeState(Collections.emptyMap(), null, 0, 1);
     KafkaProducer producer = mock(KafkaProducer.class);
     doReturn(mock(Future.class)).when(producer).send(any(ProducerRecord.class));
     Whitebox.setInternalState(state, "producer", producer);
+
     List<TridentTuple> tupleList = mockTupleList();
     for (TridentTuple t : tupleList) {
       state.updateState(Collections.singletonList(t), null);

http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-runtime/pom.xml b/external/sql/storm-sql-runtime/pom.xml
index 6663039..fd931f4 100644
--- a/external/sql/storm-sql-runtime/pom.xml
+++ b/external/sql/storm-sql-runtime/pom.xml
@@ -73,6 +73,10 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
         <!-- janino -->
         <dependency>
             <groupId>org.codehaus.janino</groupId>

http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/FieldInfo.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/FieldInfo.java b/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/FieldInfo.java
index cb1176b..03b030b 100644
--- a/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/FieldInfo.java
+++ b/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/FieldInfo.java
@@ -17,10 +17,12 @@
  */
 package org.apache.storm.sql.runtime;
 
+import java.io.Serializable;
+
 /**
  * Describe each column of the field
  */
-public class FieldInfo {
+public class FieldInfo implements Serializable {
   private final String name;
   private final Class<?> type;
   private final boolean isPrimary;

http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/ISqlTridentDataSource.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/ISqlTridentDataSource.java b/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/ISqlTridentDataSource.java
index 92961dc..9eae5ae 100644
--- a/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/ISqlTridentDataSource.java
+++ b/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/ISqlTridentDataSource.java
@@ -17,14 +17,49 @@
  */
 package org.apache.storm.sql.runtime;
 
-import org.apache.storm.trident.operation.Function;
-import org.apache.storm.trident.spout.IBatchSpout;
 import org.apache.storm.trident.spout.ITridentDataSource;
+import org.apache.storm.trident.state.StateFactory;
+import org.apache.storm.trident.state.StateUpdater;
 
 /**
  * A ISqlTridentDataSource specifies how an external data source produces and consumes data.
  */
 public interface ISqlTridentDataSource {
+  /**
+   * SqlTridentConsumer is a data structure containing StateFactory and StateUpdater for consuming tuples with State.
+   *
+   * Please note that StateFactory and StateUpdater should use same class which implements State.
+   *
+   * @see org.apache.storm.trident.state.StateFactory
+   * @see org.apache.storm.trident.state.StateUpdater
+   */
+  interface SqlTridentConsumer {
+    StateFactory getStateFactory();
+    StateUpdater getStateUpdater();
+  }
+
+  /**
+   * Provides instance of ITridentDataSource which can be used as producer in Trident.
+   *
+   * Since ITridentDataSource is a marker interface for Trident Spout interfaces, this method should effectively
+   * return an instance of one of these interfaces (can be changed if Trident API evolves) or descendants:
+   * - IBatchSpout
+   * - ITridentSpout
+   * - IPartitionedTridentSpout
+   * - IOpaquePartitionedTridentSpout
+   *
+   * @see org.apache.storm.trident.spout.ITridentDataSource
+   * @see org.apache.storm.trident.spout.IBatchSpout
+   * @see org.apache.storm.trident.spout.ITridentSpout
+   * @see org.apache.storm.trident.spout.IPartitionedTridentSpout
+   * @see org.apache.storm.trident.spout.IOpaquePartitionedTridentSpout
+   */
   ITridentDataSource getProducer();
-  Function getConsumer();
+
+  /**
+   * Provides instance of SqlTridentConsumer which can be used as consumer (State) in Trident.
+   *
+   * @see SqlTridentConsumer
+   */
+  SqlTridentConsumer getConsumer();
 }

http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/SimpleSqlTridentConsumer.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/SimpleSqlTridentConsumer.java b/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/SimpleSqlTridentConsumer.java
new file mode 100644
index 0000000..69d67d7
--- /dev/null
+++ b/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/SimpleSqlTridentConsumer.java
@@ -0,0 +1,24 @@
+package org.apache.storm.sql.runtime;
+
+import org.apache.storm.trident.state.StateFactory;
+import org.apache.storm.trident.state.StateUpdater;
+
+public class SimpleSqlTridentConsumer implements ISqlTridentDataSource.SqlTridentConsumer {
+    private final StateFactory stateFactory;
+    private final StateUpdater stateUpdater;
+
+    public SimpleSqlTridentConsumer(StateFactory stateFactory, StateUpdater stateUpdater) {
+        this.stateFactory = stateFactory;
+        this.stateUpdater = stateUpdater;
+    }
+
+    @Override
+    public StateFactory getStateFactory() {
+        return stateFactory;
+    }
+
+    @Override
+    public StateUpdater getStateUpdater() {
+        return stateUpdater;
+    }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonScheme.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonScheme.java b/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonScheme.java
new file mode 100644
index 0000000..3d4de0d
--- /dev/null
+++ b/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonScheme.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.sql.runtime.serde.json;
+
+import org.apache.storm.spout.Scheme;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.utils.Utils;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+public class JsonScheme implements Scheme {
+  private final List<String> fields;
+
+  public JsonScheme(List<String> fields) {
+    this.fields = fields;
+  }
+
+  @Override
+  public List<Object> deserialize(ByteBuffer ser) {
+    ObjectMapper mapper = new ObjectMapper();
+    try {
+      @SuppressWarnings("unchecked")
+      HashMap<String, Object> map = mapper.readValue(Utils.toByteArray(ser), HashMap.class);
+      ArrayList<Object> list = new ArrayList<>();
+      for (String f : fields) {
+        list.add(map.get(f));
+      }
+      return list;
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public Fields getOutputFields() {
+    return new Fields(fields);
+  }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonSerializer.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonSerializer.java b/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonSerializer.java
new file mode 100644
index 0000000..1e825c4
--- /dev/null
+++ b/external/sql/storm-sql-runtime/src/jvm/org/apache/storm/sql/runtime/serde/json/JsonSerializer.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.sql.runtime.serde.json;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.google.common.base.Preconditions;
+import org.apache.storm.sql.runtime.IOutputSerializer;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.io.StringWriter;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+public class JsonSerializer implements IOutputSerializer, Serializable {
+  private final List<String> fieldNames;
+  private final JsonFactory jsonFactory;
+
+  public JsonSerializer(List<String> fieldNames) {
+    this.fieldNames = fieldNames;
+    jsonFactory = new JsonFactory();
+  }
+
+  @Override
+  public ByteBuffer write(List<Object> data, ByteBuffer buffer) {
+    Preconditions.checkArgument(data != null && data.size() == fieldNames.size(), "Invalid schema");
+    StringWriter sw = new StringWriter();
+    try (JsonGenerator jg = jsonFactory.createGenerator(sw)) {
+      jg.writeStartObject();
+      for (int i = 0; i < fieldNames.size(); ++i) {
+        jg.writeFieldName(fieldNames.get(i));
+        jg.writeObject(data.get(i));
+      }
+      jg.writeEndObject();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    return ByteBuffer.wrap(sw.toString().getBytes(StandardCharsets.UTF_8));
+  }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestJsonRepresentation.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestJsonRepresentation.java b/external/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestJsonRepresentation.java
new file mode 100644
index 0000000..6ca1877
--- /dev/null
+++ b/external/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestJsonRepresentation.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.sql;
+
+import org.apache.storm.sql.runtime.serde.json.JsonScheme;
+import org.apache.storm.sql.runtime.serde.json.JsonSerializer;
+import org.apache.storm.utils.Utils;
+import com.google.common.collect.Lists;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.List;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class TestJsonRepresentation {
+  @Test
+  public void testJsonScheme() {
+    final List<String> fields = Lists.newArrayList("ID", "val");
+    final String s = "{\"ID\": 1, \"val\": \"2\"}";
+    JsonScheme scheme = new JsonScheme(fields);
+    List<Object> o = scheme.deserialize(ByteBuffer.wrap(s.getBytes(Charset.defaultCharset())));
+    assertArrayEquals(new Object[] {1, "2"}, o.toArray());
+  }
+
+  @Test
+  public void testJsonSerializer() {
+    final List<String> fields = Lists.newArrayList("ID", "val");
+    List<Object> o = Lists.<Object> newArrayList(1, "2");
+    JsonSerializer s = new JsonSerializer(fields);
+    ByteBuffer buf = s.write(o, null);
+    byte[] b = Utils.toByteArray(buf);
+    assertEquals("{\"ID\":1,\"val\":\"2\"}", new String(b));
+  }
+}

http://git-wip-us.apache.org/repos/asf/storm/blob/130b0514/external/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestUtils.java
----------------------------------------------------------------------
diff --git a/external/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestUtils.java b/external/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestUtils.java
index 58c3f26..7648f58 100644
--- a/external/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestUtils.java
+++ b/external/sql/storm-sql-runtime/src/test/org/apache/storm/sql/TestUtils.java
@@ -21,7 +21,13 @@ package org.apache.storm.sql;
 
 import org.apache.storm.ILocalCluster;
 import org.apache.storm.LocalCluster;
+import org.apache.storm.sql.runtime.SimpleSqlTridentConsumer;
+import org.apache.storm.task.IMetricsContext;
 import org.apache.storm.task.TopologyContext;
+import org.apache.storm.trident.operation.TridentOperationContext;
+import org.apache.storm.trident.state.State;
+import org.apache.storm.trident.state.StateFactory;
+import org.apache.storm.trident.state.StateUpdater;
 import org.apache.storm.tuple.Fields;
 import org.apache.storm.tuple.Values;
 import org.apache.storm.sql.runtime.ChannelContext;
@@ -188,31 +194,70 @@ public class TestUtils {
     }
   }
 
-  public static class MockSqlTridentDataSource implements ISqlTridentDataSource {
-    @Override
-    public IBatchSpout getProducer() {
-      return new MockSpout();
+  public static class MockState implements State {
+    /**
+     * Collect all values in a static variable as the instance will go through serialization and deserialization.
+     * NOTE: This should be cleared before or after running each test.
+     */
+    private transient static final List<List<Object> > VALUES = new ArrayList<>();
+
+    public static List<List<Object>> getCollectedValues() {
+      return VALUES;
     }
 
     @Override
-    public Function getConsumer() {
-      return new CollectDataFunction();
+    public void beginCommit(Long txid) {
+      // NOOP
     }
 
-    public static class CollectDataFunction extends BaseFunction {
-      /**
-       * Collect all values in a static variable as the instance will go through serialization and deserialization.
-       */
-      private transient static final List<List<Object> > VALUES = new ArrayList<>();
-      public static List<List<Object>> getCollectedValues() {
-        return VALUES;
-      }
+    @Override
+    public void commit(Long txid) {
+      // NOOP
+    }
 
-      @Override
-      public void execute(TridentTuple tuple, TridentCollector collector) {
+    public void updateState(List<TridentTuple> tuples, TridentCollector collector) {
+      for (TridentTuple tuple : tuples) {
         VALUES.add(tuple.getValues());
       }
     }
+  }
+
+  public static class MockStateFactory implements StateFactory {
+
+    @Override
+    public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
+      return new MockState();
+    }
+  }
+
+  public static class MockStateUpdater implements StateUpdater<MockState> {
+
+    @Override
+    public void updateState(MockState state, List<TridentTuple> tuples, TridentCollector collector) {
+      state.updateState(tuples, collector);
+    }
+
+    @Override
+    public void prepare(Map conf, TridentOperationContext context) {
+      // NOOP
+    }
+
+    @Override
+    public void cleanup() {
+      // NOOP
+    }
+  }
+
+  public static class MockSqlTridentDataSource implements ISqlTridentDataSource {
+    @Override
+    public IBatchSpout getProducer() {
+      return new MockSpout();
+    }
+
+    @Override
+    public SqlTridentConsumer getConsumer() {
+      return new SimpleSqlTridentConsumer(new MockStateFactory(), new MockStateUpdater());
+    }
 
     private static class MockSpout implements IBatchSpout {
       private final ArrayList<Values> RECORDS = new ArrayList<>();
@@ -269,23 +314,8 @@ public class TestUtils {
     }
 
     @Override
-    public Function getConsumer() {
-      return new CollectDataFunction();
-    }
-
-    public static class CollectDataFunction extends BaseFunction {
-      /**
-       * Collect all values in a static variable as the instance will go through serialization and deserialization.
-       */
-      private transient static final List<List<Object> > VALUES = new ArrayList<>();
-      public static List<List<Object>> getCollectedValues() {
-        return VALUES;
-      }
-
-      @Override
-      public void execute(TridentTuple tuple, TridentCollector collector) {
-        VALUES.add(tuple.getValues());
-      }
+    public SqlTridentConsumer getConsumer() {
+      return new SimpleSqlTridentConsumer(new MockStateFactory(), new MockStateUpdater());
     }
 
     private static class MockGroupedSpout implements IBatchSpout {
@@ -343,23 +373,8 @@ public class TestUtils {
     }
 
     @Override
-    public Function getConsumer() {
-      return new CollectDataFunction();
-    }
-
-    public static class CollectDataFunction extends BaseFunction {
-      /**
-       * Collect all values in a static variable as the instance will go through serialization and deserialization.
-       */
-      private transient static final List<List<Object> > VALUES = new ArrayList<>();
-      public static List<List<Object>> getCollectedValues() {
-        return VALUES;
-      }
-
-      @Override
-      public void execute(TridentTuple tuple, TridentCollector collector) {
-        VALUES.add(tuple.getValues());
-      }
+    public SqlTridentConsumer getConsumer() {
+      return new SimpleSqlTridentConsumer(new MockStateFactory(), new MockStateUpdater());
     }
 
     private static class MockSpout implements IBatchSpout {
@@ -420,23 +435,8 @@ public class TestUtils {
     }
 
     @Override
-    public Function getConsumer() {
-      return new CollectDataFunction();
-    }
-
-    public static class CollectDataFunction extends BaseFunction {
-      /**
-       * Collect all values in a static variable as the instance will go through serialization and deserialization.
-       */
-      private transient static final List<List<Object> > VALUES = new ArrayList<>();
-      public static List<List<Object>> getCollectedValues() {
-        return VALUES;
-      }
-
-      @Override
-      public void execute(TridentTuple tuple, TridentCollector collector) {
-        VALUES.add(tuple.getValues());
-      }
+    public SqlTridentConsumer getConsumer() {
+      return new SimpleSqlTridentConsumer(new MockStateFactory(), new MockStateUpdater());
     }
 
     private static class MockSpout implements IBatchSpout {
@@ -518,8 +518,4 @@ public class TestUtils {
     final long NANOSECONDS_PER_MILLISECOND = 1000000;
     return System.nanoTime() / NANOSECONDS_PER_MILLISECOND;
   }
-
-  public static ILocalCluster newLocalCluster() {
-    return new LocalCluster();
-  }
 }


[3/3] storm git commit: add STORM-2089 to CHANGELOG

Posted by ka...@apache.org.
add STORM-2089 to CHANGELOG


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8fbef379
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8fbef379
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8fbef379

Branch: refs/heads/1.x-branch
Commit: 8fbef3793cf1d18e90ec8e756ce79b1fea3ca616
Parents: c4a69c2
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Sep 27 06:52:34 2016 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Sep 27 06:52:34 2016 +0900

----------------------------------------------------------------------
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/8fbef379/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2bcfe65..e0a40b3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
 ## 1.1.0
+ * STORM-2089: Replace Consumer of ISqlTridentDataSource with SqlTridentConsumer
  * STORM-2118: A few fixes for storm-sql standalone mode
  * STORM-2105: Cluster/Supervisor total and available resources displayed in the UI
  * STORM-2078: enable paging in worker datatable


[2/3] storm git commit: Merge branch 'STORM-2089-1.x' into 1.x-branch

Posted by ka...@apache.org.
Merge branch 'STORM-2089-1.x' into 1.x-branch


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/c4a69c24
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/c4a69c24
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/c4a69c24

Branch: refs/heads/1.x-branch
Commit: c4a69c24a47f236bffc8086b57aeb78507067d75
Parents: 8bcdd7b 130b051
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Sep 27 06:51:48 2016 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Sep 27 06:51:48 2016 +0900

----------------------------------------------------------------------
 .../trident/TridentLogicalPlanCompiler.java     |   9 +-
 .../backends/trident/TestPlanCompiler.java      |  33 +++--
 .../org/apache/storm/sql/kafka/JsonScheme.java  |  58 --------
 .../apache/storm/sql/kafka/JsonSerializer.java  |  56 --------
 .../sql/kafka/KafkaDataSourcesProvider.java     |  61 +++-----
 .../storm/sql/kafka/TestJsonRepresentation.java |  50 -------
 .../sql/kafka/TestKafkaDataSourcesProvider.java |  26 +++-
 external/sql/storm-sql-runtime/pom.xml          |   4 +
 .../org/apache/storm/sql/runtime/FieldInfo.java |   4 +-
 .../sql/runtime/ISqlTridentDataSource.java      |  41 +++++-
 .../sql/runtime/SimpleSqlTridentConsumer.java   |  24 ++++
 .../sql/runtime/serde/json/JsonScheme.java      |  58 ++++++++
 .../sql/runtime/serde/json/JsonSerializer.java  |  57 ++++++++
 .../storm/sql/TestJsonRepresentation.java       |  52 +++++++
 .../test/org/apache/storm/sql/TestUtils.java    | 138 +++++++++----------
 15 files changed, 367 insertions(+), 304 deletions(-)
----------------------------------------------------------------------