You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by pt...@apache.org on 2016/09/28 20:38:05 UTC
[01/11] storm git commit: initial SQE import
Repository: storm
Updated Branches:
refs/heads/sqe_import [created] 622a80ab7
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ProcessLogicalOperatorTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ProcessLogicalOperatorTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ProcessLogicalOperatorTest.java
new file mode 100644
index 0000000..bfd641d
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ProcessLogicalOperatorTest.java
@@ -0,0 +1,121 @@
+package com.jwplayer.sqe.trident.function;
+
+import static org.junit.Assert.*;
+
+import com.jwplayer.sqe.language.expression.transform.predicate.LogicalOperatorType;
+import com.jwplayer.sqe.trident.SingleValuesCollector;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+
+public class ProcessLogicalOperatorTest {
+ @Test
+ public void testAnd() {
+ ProcessLogicalOperator operator = new ProcessLogicalOperator(LogicalOperatorType.And);
+ TridentTuple tuple;
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("LeftPredicate", "RightPredicate"), true, true);
+ operator.execute(tuple, collector);
+ assertTrue((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("LeftPredicate", "RightPredicate"), true, false);
+ operator.execute(tuple, collector);
+ assertFalse((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("LeftPredicate", "RightPredicate"), false, true);
+ operator.execute(tuple, collector);
+ assertFalse((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("LeftPredicate", "RightPredicate"), false, false);
+ operator.execute(tuple, collector);
+ assertFalse((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("P1", "P2", "P3"), true, true, true);
+ operator.execute(tuple, collector);
+ assertTrue((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("P1", "P2", "P3"), true, true, false);
+ operator.execute(tuple, collector);
+ assertFalse((Boolean) collector.values.get(0));
+ }
+
+ @Test
+ public void testNot() {
+ ProcessLogicalOperator operator = new ProcessLogicalOperator(LogicalOperatorType.Not);
+ TridentTuple tuple;
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("Predicate"), true);
+ operator.execute(tuple, collector);
+ assertFalse((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("Predicate"), false);
+ operator.execute(tuple, collector);
+ assertTrue((Boolean) collector.values.get(0));
+ }
+
+ @Test
+ public void testOr() {
+ ProcessLogicalOperator operator = new ProcessLogicalOperator(LogicalOperatorType.Or);
+ TridentTuple tuple;
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("LeftPredicate", "RightPredicate"), true, true);
+ operator.execute(tuple, collector);
+ assertTrue((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("LeftPredicate", "RightPredicate"), true, false);
+ operator.execute(tuple, collector);
+ assertTrue((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("LeftPredicate", "RightPredicate"), false, true);
+ operator.execute(tuple, collector);
+ assertTrue((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("LeftPredicate", "RightPredicate"), false, false);
+ operator.execute(tuple, collector);
+ assertFalse((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("P1", "P2", "P3"), true, false, true);
+ operator.execute(tuple, collector);
+ assertTrue((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("P1", "P2", "P3"), false, false, false);
+ operator.execute(tuple, collector);
+ assertFalse((Boolean) collector.values.get(0));
+ }
+
+ @Test
+ public void testXor() {
+ ProcessLogicalOperator operator = new ProcessLogicalOperator(LogicalOperatorType.Xor);
+ TridentTuple tuple;
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("LeftPredicate", "RightPredicate"), true, true);
+ operator.execute(tuple, collector);
+ assertFalse((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("LeftPredicate", "RightPredicate"), true, false);
+ operator.execute(tuple, collector);
+ assertTrue((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("LeftPredicate", "RightPredicate"), false, true);
+ operator.execute(tuple, collector);
+ assertTrue((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("LeftPredicate", "RightPredicate"), false, false);
+ operator.execute(tuple, collector);
+ assertFalse((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("P1", "P2", "P3"), true, true, true);
+ operator.execute(tuple, collector);
+ assertTrue((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("P1", "P2", "P3"), true, false, true);
+ operator.execute(tuple, collector);
+ assertFalse((Boolean) collector.values.get(0));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/RoundDateTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/RoundDateTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/RoundDateTest.java
new file mode 100644
index 0000000..2914b89
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/RoundDateTest.java
@@ -0,0 +1,62 @@
+package com.jwplayer.sqe.trident.function;
+
+import static org.junit.Assert.*;
+
+import com.jwplayer.sqe.trident.SingleValuesCollector;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+import java.util.Date;
+
+
+public class RoundDateTest {
+ @Test
+ public void testRoundSeconds() {
+ RoundDate roundDate = new RoundDate();
+ Date date = new Date(1000 * 105 + 5);
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("Date", "Amount", "Unit"), date, 10, "Second");
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ roundDate.execute(tuple, collector);
+
+ assertEquals(collector.values.get(0), new Date(1000 * 100));
+ }
+
+ @Test
+ public void testRoundMinutes() {
+ RoundDate roundDate = new RoundDate();
+ Date date = new Date(1000 * 60 * 105 + 1000);
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("Date", "Amount", "Unit"), date, 10, "Minute");
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ roundDate.execute(tuple, collector);
+
+ assertEquals(collector.values.get(0), new Date(1000 * 60 * 100));
+ }
+
+ @Test
+ public void testRoundHours() {
+ RoundDate roundDate = new RoundDate();
+ Date date = new Date(1000 * 60 * 60 * 105 + 1000 * 60);
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("Date", "Amount", "Unit"), date, 10, "Hour");
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ roundDate.execute(tuple, collector);
+
+ assertEquals(collector.values.get(0), new Date(1000 * 60 * 60 * 100));
+ }
+
+ @Test
+ public void testRoundDays() {
+ RoundDate roundDate = new RoundDate();
+ Date date = new Date(1000l * 60l * 60l * 24l * 105l + 1000l * 60l * 60l);
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("Date", "Amount", "Unit"), date, 10, "Day");
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ roundDate.execute(tuple, collector);
+
+ assertEquals(collector.values.get(0), new Date(1000l * 60l * 60l * 24l * 100l));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentCollectorTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentCollectorTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentCollectorTest.java
new file mode 100644
index 0000000..ff9b466
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentCollectorTest.java
@@ -0,0 +1,57 @@
+package com.jwplayer.sqe.trident.spout.kafka;
+
+import static org.junit.Assert.*;
+
+import com.jwplayer.sqe.trident.ListValuesCollector;
+import com.jwplayer.sqe.trident.StreamMetadata;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+public class FilteredTridentCollectorTest {
+ public static final long hwmTtl = 1L;
+
+ @Test
+ public void testFilter() throws InterruptedException {
+ StreamMetadata streamMetadata1 = new StreamMetadata(0, 0, 1);
+ List<Object> values1 = new ArrayList<>();
+ values1.add(streamMetadata1.toBytes());
+ StreamMetadata streamMetadata2 = new StreamMetadata(0, 0, 2);
+ List<Object> values2 = new ArrayList<>();
+ values2.add(streamMetadata2.toBytes());
+ StreamMetadata streamMetadata3 = new StreamMetadata(0, 0, 3);
+ List<Object> values3 = new ArrayList<>();
+ values3.add(streamMetadata3.toBytes());
+ ListValuesCollector collector = new ListValuesCollector();
+ FilteredTridentCollector filteredCollector = new FilteredTridentCollector(collector, 0, hwmTtl, new HashMap());
+
+ filteredCollector.emit(values1);
+ filteredCollector.emit(values2);
+ filteredCollector.emit(values1);
+ filteredCollector.emit(values2);
+ filteredCollector.emit(values3);
+ filteredCollector.emit(values2);
+ filteredCollector.emit(values3);
+
+ assertEquals(collector.values.size(), 3);
+ assertEquals(collector.values.get(0), values1);
+ assertEquals(collector.values.get(1), values2);
+ assertEquals(collector.values.get(2), values3);
+
+ Map newMetadata = new HashMap();
+ filteredCollector.resolveMetadata(newMetadata);
+
+ assertTrue(newMetadata.containsKey(FilteredTridentCollector.HIGH_WATERMARKS));
+ Map<String, Map<String, Long>> hwmMetadata = (Map) newMetadata.get(FilteredTridentCollector.HIGH_WATERMARKS);
+ assertEquals(hwmMetadata.size(), 1);
+ assertEquals(hwmMetadata.get("0-0").get(FilteredTridentCollector.WATERMARK), (Long) 3L);
+
+ TimeUnit.SECONDS.sleep(2L);
+ filteredCollector.resolveMetadata(newMetadata);
+ assertEquals(hwmMetadata.size(), 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/spout/kafka/SqeRawFullSchemeTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/spout/kafka/SqeRawFullSchemeTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/spout/kafka/SqeRawFullSchemeTest.java
new file mode 100644
index 0000000..0ddb1f3
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/spout/kafka/SqeRawFullSchemeTest.java
@@ -0,0 +1,27 @@
+package com.jwplayer.sqe.trident.spout.kafka;
+
+
+import org.apache.storm.kafka.Broker;
+import org.apache.storm.kafka.Partition;
+import org.apache.storm.kafka.ZkHosts;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+
+public class SqeRawFullSchemeTest {
+ private SqeRawFullScheme scheme;
+
+ @Before
+ public void setup() {
+ scheme = new SqeRawFullScheme("test-topo", "test-stream", new ZkHosts("brokers", "path"));
+ }
+
+ @Test
+ public void nullKeyTest() {
+ List<Object> values = scheme.deserialize(null, null, new Partition(new Broker("host"), "topic", 1), 0);
+ Assert.assertEquals(values.get(0), null);
+ Assert.assertEquals(values.get(1), null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/FileHelperTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/FileHelperTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/FileHelperTest.java
new file mode 100644
index 0000000..bca7ec8
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/FileHelperTest.java
@@ -0,0 +1,42 @@
+package com.jwplayer.sqe.util;
+
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Paths;
+
+import static junit.framework.Assert.assertNotNull;
+import static junit.framework.Assert.assertNotSame;
+
+
+public class FileHelperTest {
+ @Test
+ public void testRelativeURI() throws IOException, URISyntaxException {
+ URI uri = new URI("conf/sample-conf.yaml");
+ String text = FileHelper.loadFileAsString(uri);
+
+ assertNotNull(text);
+ assertNotSame(text, "");
+ }
+
+ @Test
+ public void testFileURI() throws IOException, URISyntaxException {
+ URI currentPath = Paths.get("").toAbsolutePath().toUri();
+ URI uri = new URI(currentPath + "conf/sample-conf.yaml");
+ String text = FileHelper.loadFileAsString(uri);
+
+ assertNotNull(text);
+ assertNotSame(text, "");
+ }
+
+ @Test
+ public void testResourceURI() throws IOException, URISyntaxException {
+ URI uri = new URI("resource:///sample-resource.txt");
+ String text = FileHelper.loadFileAsString(uri);
+
+ assertNotNull(text);
+ assertNotSame(text, "");
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/KafkaTestServer.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/KafkaTestServer.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/KafkaTestServer.java
new file mode 100644
index 0000000..e7bd2ac
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/KafkaTestServer.java
@@ -0,0 +1,107 @@
+package com.jwplayer.sqe.util;
+
+import kafka.admin.AdminUtils;
+import kafka.api.FetchRequest;
+import kafka.api.FetchRequestBuilder;
+import kafka.api.FetchResponse;
+import kafka.consumer.SimpleConsumer;
+import kafka.message.ByteBufferMessageSet;
+import kafka.message.MessageAndOffset;
+import kafka.server.KafkaConfig;
+import kafka.server.KafkaServerStartable;
+import org.apache.curator.test.InstanceSpec;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import scala.collection.Iterator;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+
+
+public class KafkaTestServer {
+ public static final String HOST = "localhost";
+ private KafkaServerStartable kafkaServer;
+ private File logDir;
+ private Integer port;
+ KafkaProducer<byte[], byte[]> producer;
+ private ZookeeperTestServer zkServer;
+
+ public KafkaTestServer() {
+ zkServer = new ZookeeperTestServer();
+ port = InstanceSpec.getRandomPort();
+ logDir = new File(System.getProperty("java.io.tmpdir"), "kafka/logs/log-" + port.toString());
+ logDir.deleteOnExit();
+ Properties kafkaProperties = new Properties();
+ kafkaProperties.setProperty("broker.id", "0");
+ kafkaProperties.setProperty("zookeeper.connect", zkServer.getConnectionString());
+ kafkaProperties.setProperty("port", port.toString());
+ kafkaProperties.setProperty("log.dirs", logDir.getAbsolutePath());
+ kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaProperties));
+ kafkaServer.startup();
+ Properties producerProperties = new Properties();
+ producerProperties.setProperty("acks", "all");
+ producerProperties.setProperty("bootstrap.servers", getConnectionString());
+ producerProperties.setProperty("key.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
+ producerProperties.setProperty("value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
+ producer = new KafkaProducer<>(producerProperties);
+ }
+
+ public void createTopic(String topic, Integer partitions) {
+ AdminUtils.createTopic(zkServer.getZkUtils(), topic, partitions, 1, new Properties());
+ }
+
+ public String getConnectionString() {
+ return HOST + ":" + port.toString();
+ }
+
+ public ZookeeperTestServer getZkServer() {
+ return zkServer;
+ }
+
+ // This one isn't working for some reason
+ public ConsumerRecords<byte[], byte[]> readMessagesBad(String topic) {
+ Properties consumerProperties = new Properties();
+ consumerProperties.setProperty("bootstrap.servers", getConnectionString());
+ consumerProperties.setProperty("group.id", "test");
+ consumerProperties.setProperty("enable.auto.commit", "false");
+ consumerProperties.setProperty("key.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer");
+ consumerProperties.setProperty("value.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer");
+ try (KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(consumerProperties)) {
+ consumer.subscribe(Arrays.asList(topic));
+ return consumer.poll(1000);
+ }
+ }
+
+ public List<MessageAndOffset> readMessages(String topic) {
+ SimpleConsumer consumer = new SimpleConsumer(HOST, port, 15000, 1024*1024, "test");
+ FetchRequestBuilder builder = new FetchRequestBuilder();
+ FetchRequest request = builder.addFetch(topic, 0, 0, 1024*1024).build();
+ FetchResponse response = consumer.fetch(request);
+ ByteBufferMessageSet messageBuffer = response.messageSet(topic, 0);
+ List<MessageAndOffset> messages = new ArrayList<>();
+ Iterator<MessageAndOffset> iter = messageBuffer.iterator();
+
+ while (iter.hasNext()) {
+ messages.add(iter.next());
+ }
+
+ return messages;
+ }
+
+ public void sendMessage(String topic, Integer partition, byte[] key, byte[] value)
+ throws ExecutionException, InterruptedException {
+ producer.send(new ProducerRecord<>(topic, partition, key, value)).get();
+ producer.flush();
+ }
+
+ public void shutdown() {
+ kafkaServer.shutdown();
+ zkServer.shutdown();
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/YamlConfigTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/YamlConfigTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/YamlConfigTest.java
new file mode 100644
index 0000000..2da8086
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/YamlConfigTest.java
@@ -0,0 +1,65 @@
+package com.jwplayer.sqe.util;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.Map;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertNotNull;
+import static junit.framework.TestCase.assertTrue;
+
+
+public class YamlConfigTest {
+ private Yaml yamlConfig;
+ private Map<String, Object> configMap;
+ private YamlConfig.InnerConfig multiTypeConfigMap;
+
+ @Before
+ public void setUp() throws IOException, URISyntaxException {
+ yamlConfig = YamlConfig.getJWConfig();
+ configMap = YamlConfig.getJWConfigAsMap();
+ multiTypeConfigMap = YamlConfig.getConfig("conf/multi-type-props.yaml");
+ }
+
+ @Test
+ public void testConfigMap() {
+ assertTrue(configMap.containsKey("SQE"));
+ }
+
+ @Test
+ public void testBoolean() {
+ assertTrue("foo", multiTypeConfigMap.getBoolean("bool", false));
+ }
+
+ @Test
+ public void testMap(){
+ assertNotNull("map", multiTypeConfigMap.getMap("map", null));
+ }
+
+ @Test
+ public void testInnerConfig() throws IOException {
+ YamlConfig.InnerConfig config = multiTypeConfigMap.getInnerConfig("map", null);
+ assertNotNull(config);
+ assertEquals(config.getInteger("hp", 0), 13);
+ assertEquals(config.getString("name", ""), "SusyTheBarbarian");
+ }
+
+ @Test
+ public void testInteger(){
+ assertEquals(multiTypeConfigMap.getInteger("int", 0), 42);
+ }
+
+ @Test
+ public void testDouble(){
+ assertEquals(multiTypeConfigMap.getDouble("float", 66.6), 3.14159);
+ }
+
+ @Test
+ public void testList(){
+ assertEquals(multiTypeConfigMap.getList("list", null).size(), 3);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/ZookeeperTestServer.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/ZookeeperTestServer.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/ZookeeperTestServer.java
new file mode 100644
index 0000000..b532686
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/ZookeeperTestServer.java
@@ -0,0 +1,48 @@
+package com.jwplayer.sqe.util;
+
+import kafka.utils.ZkUtils;
+import org.I0Itec.zkclient.ZkClient;
+import org.I0Itec.zkclient.ZkConnection;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.curator.test.TestingServer;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+
+import java.io.IOException;
+
+
+public class ZookeeperTestServer {
+ private CuratorFramework zookeeperTestServer;
+ private TestingServer testingServer;
+
+ public ZookeeperTestServer() {
+ try {
+ testingServer = new TestingServer();
+ ExponentialBackoffRetry backoffRetry = new ExponentialBackoffRetry(1000, 4);
+ zookeeperTestServer = CuratorFrameworkFactory.newClient(getConnectionString(), backoffRetry);
+ } catch (Exception ex) {
+ throw new RuntimeException("Couldn't start test ZK server", ex);
+ }
+ }
+
+ public String getConnectionString() {
+ return testingServer.getConnectString();
+ }
+
+ public ZkUtils getZkUtils() {
+ ZkClient zkClient = new ZkClient(getConnectionString());
+ return new ZkUtils(zkClient, new ZkConnection(getConnectionString()), false);
+ }
+
+ public void shutdown() {
+ try {
+ if (zookeeperTestServer.getState().equals(CuratorFrameworkState.STARTED)) {
+ zookeeperTestServer.close();
+ }
+ testingServer.close();
+ } catch (IOException ex) {
+ throw new RuntimeException("Couldn't shutdown test ZK server", ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/resources/queries/command-deserialization.json
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/resources/queries/command-deserialization.json b/external/storm-sqe/src/test/resources/queries/command-deserialization.json
new file mode 100644
index 0000000..215b9d4
--- /dev/null
+++ b/external/storm-sqe/src/test/resources/queries/command-deserialization.json
@@ -0,0 +1,71 @@
+[
+ {
+ "Set": {
+ "key": "jw.sqe.state.redis.host",
+ "value": "localhost"
+ }
+ },
+ {
+ "CreateStream": {
+ "streamName": "big.query.data",
+ "objectName": "ping-logs",
+ "spoutName": "FIXED",
+ "spoutType": "NON_TRANSACTIONAL",
+ "options": {
+ "jw.sqe.spout.fixed.fields": ["DateString", "AccountToken", "UserName", "HappyDanceCount"],
+ "jw.sqe.spout.fixed.values": [
+ ["2015-05-01 00:00", "Account1", "Joe", 1],
+ ["2015-05-01 01:00", "Account1", "Bob", -1],
+ ["2015-05-01 02:00", "Account1", "Susy", 1],
+ ["2015-05-01 03:00", "Account1", "Mr. Fancy Pants", 1],
+ ["2015-05-01 04:00", "Account1", "Joe", -1],
+ ["2015-05-01 05:00", "Account1", "Bob", 1],
+ ["2015-05-01 06:00", "Account1", "Susy", 1],
+ ["2015-05-01 07:00", "Account1", "Mr. Fancy Pants", 1],
+ ["2015-05-01 08:00", "Account2", "Joe", 1],
+ ["2015-05-01 09:00", "Account2", "Bob", 1],
+ ["2015-05-01 10:00", "Account2", "Susy", -1],
+ ["2015-05-01 11:00", "Account2", "Mr. Fancy Pants", 1],
+ ["2015-05-01 12:00", "Account2", "Joe", 1],
+ ["2015-05-01 13:00", "Account2", "Bob", 1],
+ ["2015-05-01 14:00", "Account2", "Susy", 1],
+ ["2015-05-01 15:00", "Account2", "Mr. Fancy Pants", -1]
+ ]
+ }
+ }
+ },
+ {
+ "Query": {
+ "insertInto":{
+ "objectName":"TestComplexQuery",
+ "stateName": "REDIS",
+ "stateType": "TRANSACTIONAL",
+ "fields":["DateString", "AccountToken", "UserHllBitmap", "HappyDanceCount"],
+ "options": {
+ "jw.sqe.state.redis.datatype": "HASH",
+ "jw.sqe.state.redis.keyname.fields": ["DateString"],
+ "jw.sqe.state.redis.fieldname.fields": ["AccountToken"]
+ }
+ },
+ "select":{
+ "expressions":[
+ {"FormatDate":[{"ParseDate":["DateString",{"C":"yyyy-MM-dd HH:mm"}]},{"C":"yyyy-MM-dd"}]},
+ "AccountToken",
+ {"CreateHll":["UserName"]},{"Sum":["HappyDanceCount"]}
+ ]
+ },
+ "from":{"objectName":"big.query.data"},
+ "where":
+ {"And":[
+ {"In":["UserName",{"C":"Bob"},{"C":"Susy"},{"C":"Mr. Fancy Pants"}]},
+ {"Or":[{">":["HappyDanceCount",0.0]},false]}
+ ]}
+ }
+ },
+ {
+ "Set": {
+ "key": "a",
+ "value": "b"
+ }
+ }
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/resources/queries/kafka-replays.json
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/resources/queries/kafka-replays.json b/external/storm-sqe/src/test/resources/queries/kafka-replays.json
new file mode 100644
index 0000000..de0aad2
--- /dev/null
+++ b/external/storm-sqe/src/test/resources/queries/kafka-replays.json
@@ -0,0 +1,51 @@
+[
+ {
+ "CreateStream": {
+ "streamName": "Source",
+ "objectName": "source",
+ "spoutName": "KAFKA",
+ "spoutType": "OPAQUE",
+ "deserializer": "identity",
+ "options": {
+ "jw.sqe.spout.kafka.clientid": "test-abcd",
+ "jw.sqe.spout.kafka.filterReplays": true,
+ "jw.sqe.spout.kafka.zkhosts": ["%s"]
+ }
+ }
+ },
+ {
+ "Query": {
+ "insertInto":{
+ "objectName":"middle",
+ "fields":["Message"]
+ },
+ "select":{
+ "expressions":[
+ "_value"
+ ]
+ },
+ "from":{"objectName":"Source"}
+ }
+ },
+ {
+ "Query": {
+ "insertInto":{
+ "objectName":"dest",
+ "stateName": "KAFKA",
+ "stateType": "NON_TRANSACTIONAL",
+ "fields":["Message"],
+ "options": {
+ "jw.sqe.state.kafka.keytype": "StreamMetadata",
+ "jw.sqe.state.kafka.partitionClass": "com.jwplayer.sqe.language.state.kafka.SourcePartitionPartitioner",
+ "jw.sqe.state.kafka.brokers": ["%s"]
+ }
+ },
+ "select":{
+ "expressions":[
+ "Message"
+ ]
+ },
+ "from":{"objectName":"middle"}
+ }
+ }
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/resources/queries/test-complex-query.json
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/resources/queries/test-complex-query.json b/external/storm-sqe/src/test/resources/queries/test-complex-query.json
new file mode 100644
index 0000000..f91d027
--- /dev/null
+++ b/external/storm-sqe/src/test/resources/queries/test-complex-query.json
@@ -0,0 +1,66 @@
+[
+ {
+ "Set": {
+ "key": "jw.sqe.state.redis.host",
+ "value": "analytics-redis-01.dev.longtailvideo.com"
+ }
+ },
+ {
+ "CreateStream": {
+ "streamName": "big.query.data",
+ "objectName": "ping-logs",
+ "spoutName": "FIXED",
+ "spoutType": "NON_TRANSACTIONAL",
+ "options": {
+ "jw.sqe.spout.fixed.fields": ["DateString", "AccountToken", "UserName", "HappyDanceCount"],
+ "jw.sqe.spout.fixed.values": [
+ ["2015-05-01 00:00", "Account1", "Joe", 1],
+ ["2015-05-01 01:00", "Account1", "Bob", -1],
+ ["2015-05-01 02:00", "Account1", "Susy", 1],
+ ["2015-05-01 03:00", "Account1", "Mr. Fancy Pants", 1],
+ ["2015-05-01 04:00", "Account1", "Joe", -1],
+ ["2015-05-01 05:00", "Account1", "Bob", 1],
+ ["2015-05-01 06:00", "Account1", "Susy", 1],
+ ["2015-05-01 07:00", "Account1", "Mr. Fancy Pants", 1],
+ ["2015-05-01 08:00", "Account2", "Joe", 1],
+ ["2015-05-01 09:00", "Account2", "Bob", 1],
+ ["2015-05-01 10:00", "Account2", "Susy", -1],
+ ["2015-05-01 11:00", "Account2", "Mr. Fancy Pants", 1],
+ ["2015-05-01 12:00", "Account2", "Joe", 1],
+ ["2015-05-01 13:00", "Account2", "Bob", 1],
+ ["2015-05-01 14:00", "Account2", "Susy", 1],
+ ["2015-05-01 15:00", "Account2", "Mr. Fancy Pants", -1]
+ ]
+ }
+ }
+ },
+ {
+ "Query": {
+ "insertInto":{
+ "objectName":"TestComplexQuery",
+ "stateName": "REDIS",
+ "stateType": "TRANSACTIONAL",
+ "fields":["DateString", "AccountToken", "UserHllBitmap", "HappyDanceCount"],
+ "options": {
+ "jw.sqe.state.redis.database": 3,
+ "jw.sqe.state.redis.datatype": "HASH",
+ "jw.sqe.state.redis.keyname.fields": ["DateString"],
+ "jw.sqe.state.redis.fieldname.fields": ["AccountToken"]
+ }
+ },
+ "select":{
+ "expressions":[
+ {"FormatDate":[{"ParseDate":["DateString",{"C":"yyyy-MM-dd HH:mm"}]},{"C":"yyyy-MM-dd"}]},
+ "AccountToken",
+ {"CreateHll":["UserName"]},{"Sum":["HappyDanceCount"]}
+ ]
+ },
+ "from":{"objectName":"big.query.data"},
+ "where":
+ {"And":[
+ {"In":["UserName",{"C":"Bob"},{"C":"Susy"},{"C":"Mr. Fancy Pants"}]},
+ {"Or":[{">":["HappyDanceCount",0.0]},false]}
+ ]}
+ }
+ }
+]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/resources/sample-resource.txt
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/resources/sample-resource.txt b/external/storm-sqe/src/test/resources/sample-resource.txt
new file mode 100644
index 0000000..44457f6
--- /dev/null
+++ b/external/storm-sqe/src/test/resources/sample-resource.txt
@@ -0,0 +1,5 @@
+This
+Is
+A
+Test
+File
\ No newline at end of file
[11/11] storm git commit: SQE: move storm-sqe to the bottom of the
module list since it does not compile
Posted by pt...@apache.org.
SQE: move storm-sqe to the bottom of the module list since it does not compile
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/622a80ab
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/622a80ab
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/622a80ab
Branch: refs/heads/sqe_import
Commit: 622a80ab715b8361c4d2674e26b5c44fdb0d119a
Parents: 7409c91
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Wed Sep 28 16:31:42 2016 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Wed Sep 28 16:31:42 2016 -0400
----------------------------------------------------------------------
pom.xml | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/622a80ab/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 713e17a..5955bc6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -319,7 +319,7 @@
<module>external/storm-kinesis</module>
<module>external/storm-druid</module>
<module>external/storm-submit-tools</module>
- <module>external/storm-sqe</module>
+
<!-- examples -->
<module>examples/storm-starter</module>
@@ -334,6 +334,10 @@
<module>examples/storm-hive-examples</module>
<module>examples/storm-elasticsearch-examples</module>
<module>examples/storm-mqtt-examples</module>
+
+
+ <!-- temorarily at the bottom of the build chain until compillation is fixed -->
+ <module>external/storm-sqe</module>
</modules>
<dependencies>
[02/11] storm git commit: initial SQE import
Posted by pt...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourceFileSystem.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourceFileSystem.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourceFileSystem.java
new file mode 100644
index 0000000..572ccd1
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourceFileSystem.java
@@ -0,0 +1,89 @@
+package com.jwplayer.sqe.util.file.resource;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.file.*;
+import java.nio.file.attribute.UserPrincipalLookupService;
+import java.nio.file.spi.FileSystemProvider;
+import java.util.Set;
+
+
+public class ResourceFileSystem extends FileSystem {
+ private static final String SEPARATOR = "/";
+ private final ResourceFileSystemProvider provider;
+
+ public ResourceFileSystem(ResourceFileSystemProvider provider) {
+ this.provider = provider;
+ }
+
+ @Override
+ public FileSystemProvider provider() {
+ return provider;
+ }
+
+ @Override
+ public void close() throws IOException {
+ // do nothing
+ }
+
+ @Override
+ public boolean isOpen() {
+ return true;
+ }
+
+ @Override
+ public boolean isReadOnly() {
+ return true;
+ }
+
+ @Override
+ public String getSeparator() {
+ return SEPARATOR;
+ }
+
+ @Override
+ public Iterable<Path> getRootDirectories() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Iterable<FileStore> getFileStores() {
+ return ImmutableList.of();
+ }
+
+ @Override
+ public Set<String> supportedFileAttributeViews() {
+ return ImmutableSet.of("basic");
+ }
+
+ @Override
+ public Path getPath(String first, String... more) {
+ try {
+ if (more.length == 0) {
+ return Paths.get(this.getClass().getResource(first).toURI());
+ }
+ } catch(URISyntaxException ex) {
+ throw new RuntimeException(ex);
+ }
+
+ throw new UnsupportedOperationException("getPath only supports a single path string");
+ }
+
+ @Override
+ public PathMatcher getPathMatcher(String syntaxAndPattern) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public UserPrincipalLookupService getUserPrincipalLookupService() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public WatchService newWatchService() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourceFileSystemProvider.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourceFileSystemProvider.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourceFileSystemProvider.java
new file mode 100644
index 0000000..b84f035
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourceFileSystemProvider.java
@@ -0,0 +1,108 @@
+package com.jwplayer.sqe.util.file.resource;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.file.*;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.attribute.FileAttributeView;
+import java.nio.file.spi.FileSystemProvider;
+import java.util.Map;
+import java.util.Set;
+
+
+public class ResourceFileSystemProvider extends FileSystemProvider {
+ public ResourceFileSystemProvider() {
+ }
+
+ @Override
+ public String getScheme() {
+ return "resource";
+ }
+
+ @Override
+ public FileSystem newFileSystem(URI uri, Map<String, ?> env) throws IOException {
+ return new ResourceFileSystem(this);
+ }
+
+ @Override
+ public FileSystem getFileSystem(URI uri) {
+ return new ResourceFileSystem(this);
+ }
+
+ @Override
+ public Path getPath(URI uri) {
+ return new ResourcePath(getFileSystem(uri), uri);
+ }
+
+ @Override
+ public SeekableByteChannel newByteChannel(Path path, Set<? extends OpenOption> options, FileAttribute<?>... attrs) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DirectoryStream<Path> newDirectoryStream(Path dir, DirectoryStream.Filter<? super Path> filter) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void createDirectory(Path dir, FileAttribute<?>... attrs) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void delete(Path path) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void copy(Path source, Path target, CopyOption... options) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void move(Path source, Path target, CopyOption... options) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isSameFile(Path path, Path path2) throws IOException {
+ return false;
+ }
+
+ @Override
+ public boolean isHidden(Path path) throws IOException {
+ return false;
+ }
+
+ @Override
+ public FileStore getFileStore(Path path) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void checkAccess(Path path, AccessMode... modes) throws IOException {
+ // Do nothing
+ }
+
+ @Override
+ public <V extends FileAttributeView> V getFileAttributeView(Path path, Class<V> type, LinkOption... options) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public <A extends BasicFileAttributes> A readAttributes(Path path, Class<A> type, LinkOption... options) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Map<String, Object> readAttributes(Path path, String attributes, LinkOption... options) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setAttribute(Path path, String attribute, Object value, LinkOption... options) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourcePath.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourcePath.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourcePath.java
new file mode 100644
index 0000000..0d5960a
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourcePath.java
@@ -0,0 +1,149 @@
+package com.jwplayer.sqe.util.file.resource;
+
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.*;
+import java.util.Iterator;
+
+
+public class ResourcePath implements Path {
+ URI uri;
+ FileSystem fileSystem;
+
+ public ResourcePath(FileSystem fileSystem, URI uri) {
+ this.fileSystem = fileSystem;
+ this.uri = uri;
+ }
+
+ @Override
+ public FileSystem getFileSystem() {
+ return fileSystem;
+ }
+
+ @Override
+ public boolean isAbsolute() {
+ return true;
+ }
+
+ @Override
+ public Path getRoot() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Path getFileName() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Path getParent() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getNameCount() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Path getName(int index) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Path subpath(int beginIndex, int endIndex) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean startsWith(Path other) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean startsWith(String other) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean endsWith(Path other) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean endsWith(String other) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Path normalize() {
+ return this;
+ }
+
+ @Override
+ public Path resolve(Path other) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Path resolve(String other) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Path resolveSibling(Path other) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Path resolveSibling(String other) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Path relativize(Path other) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public URI toUri() {
+ return uri;
+ }
+
+ @Override
+ public Path toAbsolutePath() {
+ return this;
+ }
+
+ @Override
+ public Path toRealPath(LinkOption... options) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public File toFile() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public WatchKey register(WatchService watcher, WatchEvent.Kind<?>[] events, WatchEvent.Modifier... modifiers) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public WatchKey register(WatchService watcher, WatchEvent.Kind<?>... events) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Iterator<Path> iterator() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int compareTo(Path other) {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/resources/META-INF/services/com.jwplayer.sqe.language.expression.FunctionExpression
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/resources/META-INF/services/com.jwplayer.sqe.language.expression.FunctionExpression b/external/storm-sqe/src/main/resources/META-INF/services/com.jwplayer.sqe.language.expression.FunctionExpression
new file mode 100644
index 0000000..0997057
--- /dev/null
+++ b/external/storm-sqe/src/main/resources/META-INF/services/com.jwplayer.sqe.language.expression.FunctionExpression
@@ -0,0 +1,30 @@
+com.jwplayer.sqe.language.expression.transform.If
+com.jwplayer.sqe.language.expression.transform.FormatDate
+com.jwplayer.sqe.language.expression.transform.GetTime
+com.jwplayer.sqe.language.expression.transform.ParseDate
+com.jwplayer.sqe.language.expression.transform.RoundDate
+com.jwplayer.sqe.language.expression.transform.ExpandKeys
+com.jwplayer.sqe.language.expression.transform.ExpandValues
+com.jwplayer.sqe.language.expression.transform.PlusArithmeticOperator
+com.jwplayer.sqe.language.expression.transform.DivisionArithmeticOperator
+com.jwplayer.sqe.language.expression.transform.ModulusArithmeticOperator
+com.jwplayer.sqe.language.expression.transform.MultiplicationArithmeticOperator
+com.jwplayer.sqe.language.expression.transform.SubtractionArithmeticOperator
+com.jwplayer.sqe.language.expression.transform.predicate.AndLogicalOperator
+com.jwplayer.sqe.language.expression.transform.predicate.In
+com.jwplayer.sqe.language.expression.transform.predicate.NotLogicalOperator
+com.jwplayer.sqe.language.expression.transform.predicate.OrLogicalOperator
+com.jwplayer.sqe.language.expression.transform.predicate.RLike
+com.jwplayer.sqe.language.expression.transform.predicate.XorLogicalOperator
+com.jwplayer.sqe.language.expression.transform.predicate.Equals
+com.jwplayer.sqe.language.expression.transform.predicate.GreaterThanCompareNumbers
+com.jwplayer.sqe.language.expression.transform.predicate.GreaterThanEqualCompareNumbers
+com.jwplayer.sqe.language.expression.transform.predicate.LessThanCompareNumbers
+com.jwplayer.sqe.language.expression.transform.predicate.LessThanEqualCompareNumbers
+com.jwplayer.sqe.language.expression.transform.predicate.NotEqualCompareNumbers
+com.jwplayer.sqe.language.expression.aggregation.CreateHll
+com.jwplayer.sqe.language.expression.aggregation.CreateHllp
+com.jwplayer.sqe.language.expression.aggregation.Max
+com.jwplayer.sqe.language.expression.aggregation.Sum
+com.jwplayer.sqe.language.expression.transform.GetMapValue
+com.jwplayer.sqe.language.expression.transform.Hash
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider b/external/storm-sqe/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider
new file mode 100644
index 0000000..de2f775
--- /dev/null
+++ b/external/storm-sqe/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider
@@ -0,0 +1 @@
+com.jwplayer.sqe.util.file.resource.ResourceFileSystemProvider
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/IntegrationTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/IntegrationTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/IntegrationTest.java
new file mode 100644
index 0000000..768a89f
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/IntegrationTest.java
@@ -0,0 +1,71 @@
+package com.jwplayer.sqe;
+
+import static org.junit.Assert.*;
+
+import com.clearspring.analytics.stream.cardinality.HyperLogLog;
+import com.jwplayer.sqe.trident.state.GsonTransactionalSerializer;
+import org.junit.Before;
+import org.junit.Test;
+import redis.clients.jedis.Jedis;
+import redis.clients.jedis.JedisPool;
+import redis.clients.jedis.JedisPoolConfig;
+import org.apache.storm.trident.state.TransactionalValue;
+
+import java.util.concurrent.TimeUnit;
+
+
+public class IntegrationTest {
+ String hostName;
+
+ @Before
+ public void setup() {
+ hostName = "..."; //TODO: This should be a test instance
+ }
+
+ public JedisPool getJedisPool() {
+ JedisPoolConfig poolConfig = new JedisPoolConfig();
+
+ poolConfig.setTestOnBorrow(true);
+ poolConfig.setTestOnReturn(true);
+ poolConfig.setMaxTotal(1);
+ poolConfig.setMaxWaitMillis(TimeUnit.SECONDS.toMillis(60));
+
+ return new JedisPool(poolConfig, hostName, 6379, 15000, null, 3);
+ }
+
+ @SuppressWarnings("unchecked")
+ //@Test
+ // This test is disabled until we can have a better way to setup test assets so we don't have collisions
+ public void testComplexQuery() throws Exception {
+ // Delete Redis keys
+ JedisPool pool = getJedisPool();
+ Jedis jedis = pool.getResource();
+ jedis.hdel("TestComplexQuery:2015-05-01", "Account1:UserHllBitmap");
+ jedis.hdel("TestComplexQuery:2015-05-01", "Account1:HappyDanceCount");
+ jedis.hdel("TestComplexQuery:2015-05-01", "Account2:UserHllBitmap");
+ jedis.hdel("TestComplexQuery:2015-05-01", "Account2:HappyDanceCount");
+
+ // Run test topologies
+ String[] args = new String[9];
+
+ args[0] = "--commands";
+ args[1] = "resource:///queries/test-complex-query.json";
+ args[2] = "--config";
+ args[3] = "./conf/sample-conf.yaml";
+ args[4] = "--local";
+ args[5] = "--name";
+ args[6] = "TestTopo";
+ args[7] = "--runtime";
+ args[8] = "120";
+
+ Topology.main(args);
+
+ // Test keys from Redis
+ GsonTransactionalSerializer serde = new GsonTransactionalSerializer();
+ TransactionalValue<byte[]> bitmapValue = serde.deserialize(jedis.hget("TestComplexQuery:2015-05-01", "Account1:UserHllBitmap").getBytes());
+ assertEquals(HyperLogLog.Builder.build(bitmapValue.getVal()).cardinality(), 3l);
+ TransactionalValue<Double> doubleValue = serde.deserialize(jedis.hget("TestComplexQuery:2015-05-01", "Account2:HappyDanceCount").getBytes());
+ assertEquals(doubleValue.getVal(), 4.0d, 0.0d); // GSON converts numbers to double by default
+ pool.returnResource(jedis);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/KafkaReplaysIntegrationTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/KafkaReplaysIntegrationTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/KafkaReplaysIntegrationTest.java
new file mode 100644
index 0000000..9942ace
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/KafkaReplaysIntegrationTest.java
@@ -0,0 +1,131 @@
+package com.jwplayer.sqe;
+
+import com.jwplayer.sqe.trident.StreamMetadata;
+import com.jwplayer.sqe.util.KafkaTestServer;
+import kafka.message.MessageAndOffset;
+import org.apache.commons.io.IOUtils;
+import org.apache.storm.utils.Utils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.PrintWriter;
+import java.util.List;
+
+
+public class KafkaReplaysIntegrationTest {
+ private final String SOURCE_TOPIC = "source";
+ private final String DEST_TOPIC = "dest";
+ private KafkaTestServer kafkaServer;
+ private List<MessageAndOffset> destMessages;
+ private List<MessageAndOffset> sourceMessages;
+
+ @Before
+ public void setup() throws Exception {
+ kafkaServer = new KafkaTestServer();
+
+ // Populate the source topic
+
+ // Partition 0
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 0, 0).toBytes(), new StreamMetadata(0L, 0, 0).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 0, 1).toBytes(), new StreamMetadata(0L, 0, 1).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 0, 2).toBytes(), new StreamMetadata(0L, 0, 2).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 0, 1).toBytes(), new StreamMetadata(0L, 0, 1).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 0, 2).toBytes(), new StreamMetadata(0L, 0, 2).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 0, 3).toBytes(), new StreamMetadata(0L, 0, 3).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 0, 2).toBytes(), new StreamMetadata(0L, 0, 2).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 0, 3).toBytes(), new StreamMetadata(0L, 0, 3).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 0, 4).toBytes(), new StreamMetadata(0L, 0, 4).toBytes());
+
+ // Partition 1
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 1, 0).toBytes(), new StreamMetadata(0L, 1, 0).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 1, 1).toBytes(), new StreamMetadata(0L, 1, 1).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 1, 2).toBytes(), new StreamMetadata(0L, 1, 2).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 1, 1).toBytes(), new StreamMetadata(0L, 1, 1).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 1, 0).toBytes(), new StreamMetadata(0L, 1, 0).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 1, 1).toBytes(), new StreamMetadata(0L, 1, 1).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 1, 2).toBytes(), new StreamMetadata(0L, 1, 2).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 1, 1).toBytes(), new StreamMetadata(0L, 1, 1).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 1, 0).toBytes(), new StreamMetadata(0L, 1, 0).toBytes());
+
+ // Partition 2
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 2, 0).toBytes(), new StreamMetadata(0L, 2, 0).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 2, 1).toBytes(), new StreamMetadata(0L, 2, 1).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 2, 2).toBytes(), new StreamMetadata(0L, 2, 2).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 2, 3).toBytes(), new StreamMetadata(0L, 2, 3).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 2, 4).toBytes(), new StreamMetadata(0L, 2, 4).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 2, 3).toBytes(), new StreamMetadata(0L, 2, 3).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 2, 2).toBytes(), new StreamMetadata(0L, 2, 2).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 2, 1).toBytes(), new StreamMetadata(0L, 2, 1).toBytes());
+ kafkaServer.sendMessage(SOURCE_TOPIC, 0, new StreamMetadata(0L, 2, 0).toBytes(), new StreamMetadata(0L, 2, 0).toBytes());
+
+ sourceMessages = kafkaServer.readMessages(SOURCE_TOPIC);
+
+ // Create command file
+ String commands = IOUtils.toString(this.getClass().getResourceAsStream("/queries/kafka-replays.json"));
+ commands = String.format(commands, kafkaServer.getZkServer().getConnectionString(), kafkaServer.getConnectionString());
+ File commandsFile = File.createTempFile("kafka-replays", "json");
+ commandsFile.deleteOnExit();
+ PrintWriter writer = new PrintWriter(commandsFile);
+ writer.println(commands);
+ writer.close();
+
+ // Run test topologies
+ String[] args = new String[9];
+
+ args[0] = "--commands";
+ args[1] = commandsFile.toURI().toString();
+ args[2] = "--config";
+ args[3] = "./conf/sample-conf.yaml";
+ args[4] = "--local";
+ args[5] = "--name";
+ args[6] = "KafkaReplays";
+ args[7] = "--runtime";
+ args[8] = "120";
+
+ Topology.main(args);
+ destMessages = kafkaServer.readMessages(DEST_TOPIC);
+ kafkaServer.shutdown();
+ }
+
+ @Test
+ public void destMessagesTest() {
+ // We expect 13 of 27 messages to make it to the destination topic
+ Assert.assertEquals(destMessages.size(), 13);
+
+ // We don't know the pid, but we can pull it from the first message. It should be consistent
+ // across all messages. We *do* know the expected ordering of the offsets from the source topic
+ // and the partition.
+ StreamMetadata metadata = StreamMetadata.parseBytes(Utils.toByteArray(destMessages.get(0).message().key()));
+ long pid = metadata.pid;
+
+ // Test each message
+ testMessage(destMessages.get(0), new StreamMetadata(pid, 0, 0).toBytes(), new StreamMetadata(0L, 0, 0).toBytes());
+ testMessage(destMessages.get(1), new StreamMetadata(pid, 0, 1).toBytes(), new StreamMetadata(0L, 0, 1).toBytes());
+ testMessage(destMessages.get(2), new StreamMetadata(pid, 0, 2).toBytes(), new StreamMetadata(0L, 0, 2).toBytes());
+ testMessage(destMessages.get(3), new StreamMetadata(pid, 0, 5).toBytes(), new StreamMetadata(0L, 0, 3).toBytes());
+ testMessage(destMessages.get(4), new StreamMetadata(pid, 0, 8).toBytes(), new StreamMetadata(0L, 0, 4).toBytes());
+ testMessage(destMessages.get(5), new StreamMetadata(pid, 0, 9).toBytes(), new StreamMetadata(0L, 1, 0).toBytes());
+ testMessage(destMessages.get(6), new StreamMetadata(pid, 0, 10).toBytes(), new StreamMetadata(0L, 1, 1).toBytes());
+ testMessage(destMessages.get(7), new StreamMetadata(pid, 0, 11).toBytes(), new StreamMetadata(0L, 1, 2).toBytes());
+ testMessage(destMessages.get(8), new StreamMetadata(pid, 0, 18).toBytes(), new StreamMetadata(0L, 2, 0).toBytes());
+ testMessage(destMessages.get(9), new StreamMetadata(pid, 0, 19).toBytes(), new StreamMetadata(0L, 2, 1).toBytes());
+ testMessage(destMessages.get(10), new StreamMetadata(pid, 0, 20).toBytes(), new StreamMetadata(0L, 2, 2).toBytes());
+ testMessage(destMessages.get(11), new StreamMetadata(pid, 0, 21).toBytes(), new StreamMetadata(0L, 2, 3).toBytes());
+ testMessage(destMessages.get(12), new StreamMetadata(pid, 0, 22).toBytes(), new StreamMetadata(0L, 2, 4).toBytes());
+ }
+
+ @Test
+ public void soureMessagesTest() {
+ Assert.assertEquals(sourceMessages.size(), 27);
+ }
+
+ public void testMessage(MessageAndOffset messageAndOffset, byte[] expectedKey, byte[] expectedValue) {
+ byte[] key = Utils.toByteArray(messageAndOffset.message().key());
+ byte[] value = Utils.toByteArray(messageAndOffset.message().payload());
+
+ Assert.assertArrayEquals(expectedKey, key);
+ Assert.assertArrayEquals(expectedValue, value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/command/CommandDeserializationTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/command/CommandDeserializationTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/command/CommandDeserializationTest.java
new file mode 100644
index 0000000..8b5ee25
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/command/CommandDeserializationTest.java
@@ -0,0 +1,49 @@
+package com.jwplayer.sqe.language.command;
+
+import static org.junit.Assert.*;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+
+public class CommandDeserializationTest {
+ List<BaseCommand> commands;
+
+ @Before
+ public void setup() throws IOException {
+ String json = IOUtils.toString(this.getClass().getResourceAsStream("/queries/command-deserialization.json"));
+ commands = BaseCommand.load(json);
+ }
+
+ @Test
+ public void testLoadCommand() {
+ assertEquals(commands.size(), 4);
+ }
+
+ @Test
+ public void testSetCommand() {
+ assertEquals(commands.get(0).getCommandType(), CommandType.Set);
+
+ SetCommand setCommand = (SetCommand) commands.get(0);
+
+ assertEquals(setCommand.key, "jw.sqe.state.redis.host");
+ assertEquals(setCommand.value, "localhost");
+ }
+
+ @Test
+ public void testQueryCommand() {
+ assertEquals(commands.get(2).getCommandType(), CommandType.Query);
+
+ Query query = (Query) commands.get(2);
+
+ assertNotNull(query.insertInto);
+ assertEquals(query.insertInto.objectName, "TestComplexQuery");
+ assertNotNull(query.select);
+ assertEquals(query.select.expressions.size(), 4);
+ assertNotNull(query.from.objectName, "big.query.data");
+ assertNotNull(query.where);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/expression/ExpressionDeserializationTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/expression/ExpressionDeserializationTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/expression/ExpressionDeserializationTest.java
new file mode 100644
index 0000000..b483443
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/expression/ExpressionDeserializationTest.java
@@ -0,0 +1,103 @@
+package com.jwplayer.sqe.language.expression;
+
+import com.google.gson.GsonBuilder;
+import static org.junit.Assert.*;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ExpressionDeserializationTest {
+ GsonBuilder gsonBuilder = null;
+
+ @Before
+ public void setup() {
+ gsonBuilder = new GsonBuilder();
+ gsonBuilder.registerTypeAdapter(BaseExpression.class, new BaseExpression.BaseExpressionTypeAdapter());
+ }
+
+ @After
+ public void shutdown() {
+
+ }
+
+ @Test
+ public void testBoolean() {
+ BaseExpression boolConstant1 = gsonBuilder.create().fromJson("{\"C\":true}", BaseExpression.class);
+ BaseExpression boolConstant2 = gsonBuilder.create().fromJson("false", BaseExpression.class);
+
+ assertTrue(boolConstant1 instanceof ConstantExpression);
+ assertTrue(((ConstantExpression) boolConstant1).constant instanceof Boolean);
+ assertTrue((Boolean) ((ConstantExpression) boolConstant1).constant);
+ assertTrue(boolConstant2 instanceof ConstantExpression);
+ assertTrue(((ConstantExpression) boolConstant2).constant instanceof Boolean);
+ assertTrue(!(Boolean) ((ConstantExpression) boolConstant2).constant);
+
+ }
+
+ @Test
+ public void testField() {
+ BaseExpression field = gsonBuilder.create().fromJson("test", BaseExpression.class);
+
+ assertTrue(field instanceof FieldExpression);
+ assertTrue(((FieldExpression) field).fieldName.equals("test"));
+ }
+
+ @Test
+ public void testFunction() {
+ BaseExpression ifExpression = gsonBuilder.create().fromJson("{\"If\":[{\">\":[\"Plays\",0]},1,0]}", BaseExpression.class);
+ BaseExpression sumExpression = gsonBuilder.create().fromJson("{\"Sum\":[\"Embeds\"]}", BaseExpression.class);
+
+ assertTrue(ifExpression instanceof FunctionExpression);
+ assertTrue(((FunctionExpression) ifExpression).getFunctionName().equals("if"));
+ assertTrue(sumExpression instanceof FunctionExpression);
+ assertTrue(((FunctionExpression) sumExpression).getFunctionName().equals("sum"));
+ }
+
+ @Test
+ public void testNull() {
+ BaseExpression nullConstant1 = gsonBuilder.create().fromJson("{\"C\":null}", BaseExpression.class);
+ // This works as part of the arguments to a function, but doesn't work here for some reason
+ //BaseExpression nullConstant2 = gsonBuilder.create().fromJson("null", BaseExpression.class);
+
+ assertTrue(nullConstant1 instanceof ConstantExpression);
+ assertNull(((ConstantExpression) nullConstant1).constant);
+ //assertTrue(nullConstant2 instanceof ConstantExpression);
+ //assertNull(((ConstantExpression) nullConstant2).constant);
+ }
+
+ @Test
+ public void testDouble() {
+ BaseExpression doubleConstant = gsonBuilder.create().fromJson("1.1", BaseExpression.class);
+
+ assertTrue(doubleConstant instanceof ConstantExpression);
+ assertTrue(((ConstantExpression) doubleConstant).constant instanceof Double);
+ assertTrue((Double) ((ConstantExpression) doubleConstant).constant == 1.1);
+ }
+
+ @Test
+ public void testInteger() {
+ BaseExpression integerConstant = gsonBuilder.create().fromJson("{\"C\":0}", BaseExpression.class);
+
+ assertTrue(integerConstant instanceof ConstantExpression);
+ assertTrue(((ConstantExpression) integerConstant).constant instanceof Integer);
+ assertTrue((Integer) ((ConstantExpression) integerConstant).constant == 0);
+ }
+
+ @Test
+ public void testLong() {
+ BaseExpression longConstant = gsonBuilder.create().fromJson("{\"C\":999999999999}", BaseExpression.class);
+
+ assertTrue(longConstant instanceof ConstantExpression);
+ assertTrue(((ConstantExpression) longConstant).constant instanceof Long);
+ assertTrue((Long) ((ConstantExpression) longConstant).constant == 999999999999l);
+ }
+
+ @Test
+ public void testString() {
+ BaseExpression stringConstant = gsonBuilder.create().fromJson("{\"C\":\"test\"}", BaseExpression.class);
+
+ assertTrue(stringConstant instanceof ConstantExpression);
+ assertTrue(((ConstantExpression) stringConstant).constant instanceof String);
+ assertTrue(((ConstantExpression) stringConstant).constant.equals("test"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/expression/FunctionExpressionTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/expression/FunctionExpressionTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/expression/FunctionExpressionTest.java
new file mode 100644
index 0000000..52ee8ef
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/expression/FunctionExpressionTest.java
@@ -0,0 +1,46 @@
+package com.jwplayer.sqe.language.expression;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.google.gson.GsonBuilder;
+
+import static org.junit.Assert.*;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class FunctionExpressionTest {
+ GsonBuilder gsonBuilder = null;
+
+ @Before
+ public void setup() {
+ gsonBuilder = new GsonBuilder();
+ gsonBuilder.registerTypeAdapter(BaseExpression.class,
+ new BaseExpression.BaseExpressionTypeAdapter());
+ }
+
+ @After
+ public void shutdown() {
+
+ }
+
+ @Test
+ public void testMakeFunction() {
+
+ List<BaseExpression> arguments = new ArrayList<>();
+
+ BaseExpression integerConstant1 = gsonBuilder.create().fromJson("{\"C\":0}", BaseExpression.class);
+ arguments.add(integerConstant1);
+ BaseExpression integerConstant2 = gsonBuilder.create().fromJson("{\"C\":1}", BaseExpression.class);
+ arguments.add(integerConstant2);
+ FunctionExpression test = FunctionExpression.makeFunction("+", arguments);
+ assertEquals(test.getClass().toString(),
+ "class com.jwplayer.sqe.language.expression.transform.PlusArithmeticOperator");
+
+ assertEquals(test.getFunctionName(), "+");
+
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/GetTuplesFilter.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/GetTuplesFilter.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/GetTuplesFilter.java
new file mode 100644
index 0000000..46361ae
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/GetTuplesFilter.java
@@ -0,0 +1,23 @@
+package com.jwplayer.sqe.trident;
+
+import org.apache.storm.trident.operation.BaseFilter;
+import org.apache.storm.trident.tuple.TridentTuple;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class GetTuplesFilter extends BaseFilter {
+ public List<TridentTuple> tuples;
+
+ public GetTuplesFilter() {
+ this.tuples = new ArrayList<>();
+ }
+
+ public boolean isKeep(TridentTuple tuple) {
+ System.out.println("TUPLES: " + tuple.toString());
+ tuples.add(tuple);
+ System.out.println("TUPLES: " + tuples.size());
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/GetMaximumAggregatorTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/GetMaximumAggregatorTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/GetMaximumAggregatorTest.java
new file mode 100644
index 0000000..5dc7eb6
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/GetMaximumAggregatorTest.java
@@ -0,0 +1,64 @@
+package com.jwplayer.sqe.trident.aggregator;
+
+import static org.junit.Assert.*;
+
+import com.jwplayer.sqe.trident.SingleValuesCollector;
+import org.junit.Before;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+
+
+public class GetMaximumAggregatorTest {
+ private GetMaximumAggregator aggregator;
+ private SingleValuesCollector collector;
+
+ @Before
+ public void setup() {
+ aggregator = new GetMaximumAggregator();
+ collector = new SingleValuesCollector();
+ }
+
+ @Test
+ public void testNumbers() {
+ GetMaximumAggregator.MaximumState state = aggregator.init(null, collector);
+
+ assertEquals(state.comparable, null);
+
+ aggregator.aggregate(state, TridentTupleView.createFreshTuple(new Fields("Object"), 1), collector);
+ aggregator.aggregate(state, TridentTupleView.createFreshTuple(new Fields("Object"), 2.0f), collector);
+ aggregator.aggregate(state, TridentTupleView.createFreshTuple(new Fields("Object"), 3l), collector);
+
+ assertEquals(state.comparable, 3l);
+
+ state.comparable = aggregator.combine(state.comparable, null);
+
+ assertEquals(state.comparable, 3l);
+
+ aggregator.complete(state, collector);
+
+ assertEquals(collector.values.get(0), 3l);
+ }
+
+ @Test
+ public void testStrings() {
+ GetMaximumAggregator.MaximumState state = aggregator.init(null, collector);
+
+ assertEquals(state.comparable, null);
+
+ aggregator.aggregate(state, TridentTupleView.createFreshTuple(new Fields("Object"), "a"), collector);
+ aggregator.aggregate(state, TridentTupleView.createFreshTuple(new Fields("Object"), "b"), collector);
+ aggregator.aggregate(state, TridentTupleView.createFreshTuple(new Fields("Object"), "c"), collector);
+
+ assertEquals(state.comparable, "c");
+
+ state.comparable = aggregator.combine(state.comparable, null);
+
+ assertEquals(state.comparable, "c");
+
+ aggregator.complete(state, collector);
+
+ assertEquals(collector.values.get(0), "c");
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllAggregatorTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllAggregatorTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllAggregatorTest.java
new file mode 100644
index 0000000..7d8e826
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllAggregatorTest.java
@@ -0,0 +1,43 @@
+package com.jwplayer.sqe.trident.aggregator;
+
+import static org.junit.Assert.*;
+
+import com.clearspring.analytics.stream.cardinality.HyperLogLog;
+import com.clearspring.analytics.stream.cardinality.ICardinality;
+import com.jwplayer.sqe.trident.SingleValuesCollector;
+import org.junit.Before;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+import java.io.IOException;
+
+
+public class HllAggregatorTest {
+ private CardinalityEstimatorAggregator aggregator;
+ private SingleValuesCollector collector;
+
+ @Before
+ public void setup() {
+ aggregator = new HllAggregator(11);
+ collector = new SingleValuesCollector();
+ }
+
+ @Test
+ public void testAggregator() throws IOException {
+ ICardinality hll = aggregator.init(null, collector);
+
+ assertEquals(hll.cardinality(), 0);
+
+ aggregator.aggregate(hll, TridentTupleView.createFreshTuple(new Fields("Object"), "a"), collector);
+ aggregator.aggregate(hll, TridentTupleView.createFreshTuple(new Fields("Object"), "b"), collector);
+ aggregator.aggregate(hll, TridentTupleView.createFreshTuple(new Fields("Object"), "c"), collector);
+
+ assertEquals(hll.cardinality(), 3);
+
+ aggregator.complete(hll, collector);
+ HyperLogLog hll2 = HyperLogLog.Builder.build((byte[]) collector.values.get(0));
+
+ assertEquals(hll.cardinality(), hll2.cardinality());
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllCombinerAggregatorTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllCombinerAggregatorTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllCombinerAggregatorTest.java
new file mode 100644
index 0000000..c616a7a
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllCombinerAggregatorTest.java
@@ -0,0 +1,45 @@
+package com.jwplayer.sqe.trident.aggregator;
+
+import static org.junit.Assert.*;
+
+import com.clearspring.analytics.stream.cardinality.HyperLogLog;
+import org.junit.Before;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+
+public class HllCombinerAggregatorTest {
+ private CardinalityEstimatorCombinerAggregator aggregator;
+
+ @Before
+ public void setup() throws IOException {
+ aggregator = new HllCombinerAggregator(11);
+ }
+
+ @Test
+ public void testAggregator() throws IOException {
+ HyperLogLog hll = new HyperLogLog(11);
+
+ hll.offer("a");
+ hll.offer("b");
+ hll.offer("c");
+
+ byte[] hllBytes = aggregator.zero();
+
+ assertEquals(HyperLogLog.Builder.build(hllBytes).cardinality(), 0);
+
+ List<Object> list = new ArrayList<>();
+ list.add(hllBytes);
+
+ assertEquals(HyperLogLog.Builder.build(aggregator.init(TridentTupleView.createFreshTuple(new Fields("hll"), list))).cardinality(), 0);
+
+ hllBytes = aggregator.combine(hllBytes, hll.getBytes());
+
+ assertEquals(HyperLogLog.Builder.build(hllBytes).cardinality(), 3);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllpAggregatorTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllpAggregatorTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllpAggregatorTest.java
new file mode 100644
index 0000000..03705b2
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllpAggregatorTest.java
@@ -0,0 +1,43 @@
+package com.jwplayer.sqe.trident.aggregator;
+
+import static org.junit.Assert.*;
+
+import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
+import com.clearspring.analytics.stream.cardinality.ICardinality;
+import com.jwplayer.sqe.trident.SingleValuesCollector;
+import org.junit.Before;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+import java.io.IOException;
+
+
+public class HllpAggregatorTest {
+ private CardinalityEstimatorAggregator aggregator;
+ private SingleValuesCollector collector;
+
+ @Before
+ public void setup() {
+ aggregator = new HllpAggregator(11);
+ collector = new SingleValuesCollector();
+ }
+
+ @Test
+ public void testAggregator() throws IOException {
+ ICardinality hllp = aggregator.init(null, collector);
+
+ assertEquals(hllp.cardinality(), 0);
+
+ aggregator.aggregate(hllp, TridentTupleView.createFreshTuple(new Fields("Object"), "a"), collector);
+ aggregator.aggregate(hllp, TridentTupleView.createFreshTuple(new Fields("Object"), "b"), collector);
+ aggregator.aggregate(hllp, TridentTupleView.createFreshTuple(new Fields("Object"), "c"), collector);
+
+ assertEquals(hllp.cardinality(), 3);
+
+ aggregator.complete(hllp, collector);
+ HyperLogLogPlus hllp2 = HyperLogLogPlus.Builder.build((byte[]) collector.values.get(0));
+
+ assertEquals(hllp.cardinality(), hllp2.cardinality());
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllpCombinerAggregatorTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllpCombinerAggregatorTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllpCombinerAggregatorTest.java
new file mode 100644
index 0000000..5e59af5
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllpCombinerAggregatorTest.java
@@ -0,0 +1,45 @@
+package com.jwplayer.sqe.trident.aggregator;
+
+import static org.junit.Assert.*;
+
+import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
+import org.junit.Before;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+
+public class HllpCombinerAggregatorTest {
+ private CardinalityEstimatorCombinerAggregator aggregator;
+
+ @Before
+ public void setup() throws IOException {
+ aggregator = new HllpCombinerAggregator(11);
+ }
+
+ @Test
+ public void testAggregator() throws IOException {
+ HyperLogLogPlus hllp = new HyperLogLogPlus(11);
+
+ hllp.offer("a");
+ hllp.offer("b");
+ hllp.offer("c");
+
+ byte[] hllpBytes = aggregator.zero();
+
+ assertEquals(HyperLogLogPlus.Builder.build(hllpBytes).cardinality(), 0);
+
+ List<Object> list = new ArrayList<>();
+ list.add(hllpBytes);
+
+ assertEquals(HyperLogLogPlus.Builder.build(aggregator.init(TridentTupleView.createFreshTuple(new Fields("hllp"), list))).cardinality(), 0);
+
+ hllpBytes = aggregator.combine(hllpBytes, hllp.getBytes());
+
+ assertEquals(HyperLogLogPlus.Builder.build(hllpBytes).cardinality(), 3);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/AddFieldTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/AddFieldTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/AddFieldTest.java
new file mode 100644
index 0000000..0767e55
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/AddFieldTest.java
@@ -0,0 +1,52 @@
+package com.jwplayer.sqe.trident.function;
+
+import static org.junit.Assert.*;
+
+import com.clearspring.analytics.stream.cardinality.HyperLogLog;
+import com.jwplayer.sqe.trident.SingleValuesCollector;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+
+public class AddFieldTest {
+ @Test
+ public void testAddInteger() {
+ AddField addField = new AddField(1);
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields());
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ addField.execute(tuple, collector);
+
+ assertEquals(collector.values.get(0), 1);
+ }
+
+ @Test
+ public void testAddString() {
+ AddField addField = new AddField("Abc");
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields());
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ addField.execute(tuple, collector);
+
+ assertEquals(collector.values.get(0), "Abc");
+ }
+
+ @Test
+ public void testAddBitmap() throws IOException {
+ HyperLogLog hll = new HyperLogLog(9);
+ hll.offer("a");
+ hll.offer("b");
+ hll.offer("C");
+ AddField addField = new AddField(hll.getBytes());
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields());
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ addField.execute(tuple, collector);
+ assertTrue(Arrays.equals((byte[]) collector.values.get(0), hll.getBytes()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/CompareNumbersTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/CompareNumbersTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/CompareNumbersTest.java
new file mode 100644
index 0000000..137689d
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/CompareNumbersTest.java
@@ -0,0 +1,76 @@
+package com.jwplayer.sqe.trident.function;
+
+import static org.junit.Assert.*;
+
+import com.jwplayer.sqe.language.expression.transform.predicate.NumberComparisonType;
+import com.jwplayer.sqe.trident.SingleValuesCollector;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+
+public class CompareNumbersTest {
+ @Test
+ public void testEquals() {
+ performTests(new CompareNumbers(NumberComparisonType.Equal), true);
+ }
+
+ @Test
+ public void testGreaterThan() {
+ performTests(new CompareNumbers(NumberComparisonType.GreaterThan), false);
+ }
+
+ @Test
+ public void testGreaterThanOrEqual() {
+ performTests(new CompareNumbers(NumberComparisonType.GreaterThanOrEqual), true);
+ }
+
+ @Test
+ public void testLessThan() {
+ performTests(new CompareNumbers(NumberComparisonType.LessThan), false);
+ }
+
+ @Test
+ public void testLessThanOrEqual() {
+ performTests(new CompareNumbers(NumberComparisonType.LessThanOrEqual), true);
+ }
+
+ @Test
+ public void testNotEqual() {
+ performTests(new CompareNumbers(NumberComparisonType.NotEqual), false);
+ }
+
+ private void performTests(CompareNumbers compareNumbers, Boolean assertTrue) {
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ compareNumbers.execute(TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 1, 1), collector);
+ assertTrue(assertTrue ? (Boolean) collector.values.get(0) : !(Boolean) collector.values.get(0));
+
+ compareNumbers.execute(TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 1, 1l), collector);
+ assertTrue(assertTrue ? (Boolean) collector.values.get(0) : !(Boolean) collector.values.get(0));
+
+ compareNumbers.execute(TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 1, 1.0f), collector);
+ assertTrue(assertTrue ? (Boolean) collector.values.get(0) : !(Boolean) collector.values.get(0));
+
+ compareNumbers.execute(TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 1, 1.0d), collector);
+ assertTrue(assertTrue ? (Boolean) collector.values.get(0) : !(Boolean) collector.values.get(0));
+
+ compareNumbers.execute(TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 1l, 1l), collector);
+ assertTrue(assertTrue ? (Boolean) collector.values.get(0) : !(Boolean) collector.values.get(0));
+
+ compareNumbers.execute(TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 1l, 1.0f), collector);
+ assertTrue(assertTrue ? (Boolean) collector.values.get(0) : !(Boolean) collector.values.get(0));
+
+ compareNumbers.execute(TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 1l, 1.0d), collector);
+ assertTrue(assertTrue ? (Boolean) collector.values.get(0) : !(Boolean) collector.values.get(0));
+
+ compareNumbers.execute(TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 1.0f, 1.0f), collector);
+ assertTrue(assertTrue ? (Boolean) collector.values.get(0) : !(Boolean) collector.values.get(0));
+
+ compareNumbers.execute(TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 1.0f, 1.0d), collector);
+ assertTrue(assertTrue ? (Boolean) collector.values.get(0) : !(Boolean) collector.values.get(0));
+
+ compareNumbers.execute(TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 1.0d, 1.0d), collector);
+ assertTrue(assertTrue ? (Boolean) collector.values.get(0) : !(Boolean) collector.values.get(0));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/CompareObjectsTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/CompareObjectsTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/CompareObjectsTest.java
new file mode 100644
index 0000000..07c84ae
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/CompareObjectsTest.java
@@ -0,0 +1,81 @@
+package com.jwplayer.sqe.trident.function;
+
+import static org.junit.Assert.*;
+
+import com.jwplayer.sqe.trident.SingleValuesCollector;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+
+
+public class CompareObjectsTest {
+ @Test
+ public void testNullComparison() {
+ CompareObjects comparator = new CompareObjects();
+ TridentTuple tuple;
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("S1", "S2"), null, 1);
+ comparator.execute(tuple, collector);
+ assertFalse((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("S1", "S2"), 1, null);
+ comparator.execute(tuple, collector);
+ assertFalse((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("S1", "S2"), null, null);
+ comparator.execute(tuple, collector);
+ assertTrue((Boolean) collector.values.get(0));
+ }
+
+ @Test
+ public void testNumberComparison() {
+ CompareObjects comparator = new CompareObjects();
+ TridentTuple tuple;
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("S1", "S2"), 1, 1.0f);
+ comparator.execute(tuple, collector);
+ assertTrue((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("S1", "S2"), 1, 2);
+ comparator.execute(tuple, collector);
+ assertFalse((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("S1", "S2"), 1, "Cat");
+ comparator.execute(tuple, collector);
+ assertFalse((Boolean) collector.values.get(0));
+ }
+
+ @Test
+ public void testObjectComparison() {
+ CompareObjects comparator = new CompareObjects();
+ TridentTuple tuple;
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("S1", "S2"), comparator, comparator);
+ comparator.execute(tuple, collector);
+ assertTrue((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("S1", "S2"), comparator, new CompareObjects());
+ comparator.execute(tuple, collector);
+ assertFalse((Boolean) collector.values.get(0));
+ }
+
+ @Test
+ public void testStringComparison() {
+ CompareObjects comparator = new CompareObjects();
+ TridentTuple tuple;
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("S1", "S2"), "Cat", "Cat");
+ comparator.execute(tuple, collector);
+ assertTrue((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("S1", "S2"), "Cat", "Dog");
+ comparator.execute(tuple, collector);
+ assertFalse((Boolean) collector.values.get(0));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/EvaluateRegularExpressionTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/EvaluateRegularExpressionTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/EvaluateRegularExpressionTest.java
new file mode 100644
index 0000000..e081bce
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/EvaluateRegularExpressionTest.java
@@ -0,0 +1,31 @@
+package com.jwplayer.sqe.trident.function;
+
+import static org.junit.Assert.*;
+
+import com.jwplayer.sqe.trident.SingleValuesCollector;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+
+public class EvaluateRegularExpressionTest {
+ @Test
+ public void testEvaluation() {
+ EvaluateRegularExpression evaluator = new EvaluateRegularExpression();
+ TridentTuple tuple;
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("Value", "RegEx"), "ABCD1234", "[0-9a-zA-Z]{8}");
+ evaluator.execute(tuple, collector);
+ assertTrue((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("Value", "RegEx"), "ABC_1234", "[0-9a-zA-Z]{8}");
+ evaluator.execute(tuple, collector);
+ assertFalse((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("Value", "RegEx"), "ABCD12345", "[0-9a-zA-Z]{8}");
+ evaluator.execute(tuple, collector);
+ assertFalse((Boolean) collector.values.get(0));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ExpandKeysTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ExpandKeysTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ExpandKeysTest.java
new file mode 100644
index 0000000..3688b78
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ExpandKeysTest.java
@@ -0,0 +1,37 @@
+package com.jwplayer.sqe.trident.function;
+
+import com.jwplayer.sqe.trident.ListValuesCollector;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+
+
+public class ExpandKeysTest {
+ @Test
+ public void testExpand() {
+ ExpandKeys expandKeys = new ExpandKeys();
+ Map<String, Integer> map = new HashMap<>();
+
+ map.put("A", 1);
+ map.put("B", 2);
+ map.put("C", 3);
+ map.put("D", 4);
+
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("Map"), map);
+ ListValuesCollector collector = new ListValuesCollector();
+
+ expandKeys.execute(tuple, collector);
+
+ assertEquals(collector.values.size(), 4);
+ assertEquals(collector.values.get(0).get(0), "A");
+ assertEquals(collector.values.get(1).get(0), "B");
+ assertEquals(collector.values.get(2).get(0), "C");
+ assertEquals(collector.values.get(3).get(0), "D");
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ExpandValuesTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ExpandValuesTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ExpandValuesTest.java
new file mode 100644
index 0000000..45e374c
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ExpandValuesTest.java
@@ -0,0 +1,37 @@
+package com.jwplayer.sqe.trident.function;
+
+import com.jwplayer.sqe.trident.ListValuesCollector;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+import java.util.HashMap;
+import java.util.Map;
+
+
+import static org.junit.Assert.*;
+
+public class ExpandValuesTest {
+ @Test
+ public void testExpand() {
+ ExpandValues expandValues = new ExpandValues();
+ Map<String, Integer> map = new HashMap<>();
+
+ map.put("A", 1);
+ map.put("B", 2);
+ map.put("C", 3);
+ map.put("D", 4);
+
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("Map"), map);
+ ListValuesCollector collector = new ListValuesCollector();
+
+ expandValues.execute(tuple, collector);
+
+ assertEquals(collector.values.size(), 4);
+ assertEquals(collector.values.get(0).get(0), 1);
+ assertEquals(collector.values.get(1).get(0), 2);
+ assertEquals(collector.values.get(2).get(0), 3);
+ assertEquals(collector.values.get(3).get(0), 4);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/FormatDateTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/FormatDateTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/FormatDateTest.java
new file mode 100644
index 0000000..776a9c2
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/FormatDateTest.java
@@ -0,0 +1,27 @@
+package com.jwplayer.sqe.trident.function;
+
+import static org.junit.Assert.*;
+
+import com.jwplayer.sqe.trident.SingleValuesCollector;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+import java.util.Date;
+
+
+public class FormatDateTest {
+ @Test
+ public void testFormatDate() {
+ FormatDate formatDate = new FormatDate();
+ Date date = new Date(0l);
+ String format = "yyyy-MM-dd HH:mm:ss";
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("Date", "Format"), date, format);
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ formatDate.execute(tuple, collector);
+
+ assertEquals(collector.values.get(0), "1970-01-01 00:00:00");
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetCardinalityEstimationTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetCardinalityEstimationTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetCardinalityEstimationTest.java
new file mode 100644
index 0000000..73ab2d3
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetCardinalityEstimationTest.java
@@ -0,0 +1,48 @@
+package com.jwplayer.sqe.trident.function;
+
+import com.clearspring.analytics.stream.cardinality.HyperLogLog;
+import com.jwplayer.sqe.trident.SingleValuesCollector;
+import org.junit.Before;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+
+
+public class GetCardinalityEstimationTest {
+ GetCardinalityEstimation estimator = null;
+ HyperLogLog hll = null;
+
+ @Before
+ public void setup() {
+ estimator = new GetCardinalityEstimation();
+ hll = new HyperLogLog(11);
+ hll.offer("a");
+ hll.offer("b");
+ hll.offer("C");
+ }
+
+ @Test
+ public void testHyperLogLogEstimation() {
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("hll"), hll);
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ estimator.execute(tuple, collector);
+
+ assertEquals(collector.values.get(0), 3l);
+ }
+
+ @Test
+ public void testBitmapEstimation() throws IOException {
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("hll"), hll.getBytes());
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ estimator.execute(tuple, collector);
+
+ assertEquals(collector.values.get(0), 3l);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetIfTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetIfTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetIfTest.java
new file mode 100644
index 0000000..730a8b8
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetIfTest.java
@@ -0,0 +1,31 @@
+package com.jwplayer.sqe.trident.function;
+
+import com.jwplayer.sqe.trident.SingleValuesCollector;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+import static org.junit.Assert.assertEquals;
+
+public class GetIfTest {
+ @Test
+ public void testFalsePredicate() {
+ GetIf getIf = new GetIf();
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("Predicate", "TrueValue", "FalseValue"), false, "a", "b");
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ getIf.execute(tuple, collector);
+ assertEquals(collector.values.get(0), "b");
+ }
+
+ @Test
+ public void testTruePredicate() {
+ GetIf getIf = new GetIf();
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("Predicate", "TrueValue", "FalseValue"), true, "a", "b");
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ getIf.execute(tuple, collector);
+ assertEquals(collector.values.get(0), "a");
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetMapValueTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetMapValueTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetMapValueTest.java
new file mode 100644
index 0000000..e5420e4
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetMapValueTest.java
@@ -0,0 +1,36 @@
+package com.jwplayer.sqe.trident.function;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Test;
+
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+import com.jwplayer.sqe.trident.SingleValuesCollector;
+
+
+public class GetMapValueTest {
+ @Test
+ public void testGetMapValue() {
+ GetMapValue getMapValueFromKey = new GetMapValue();
+ Map<Integer, String> map = new HashMap<Integer, String>();
+ map.put(123, "S");
+ map.put(456, "T");
+
+ TridentTuple tuple1 = TridentTupleView.createFreshTuple(new Fields("map", "key"), map, 123);
+ TridentTuple tuple2 = TridentTupleView.createFreshTuple(new Fields("map", "key"), map, 456);
+
+ SingleValuesCollector collector = new SingleValuesCollector();
+ getMapValueFromKey.execute(tuple1, collector);
+ assertEquals(collector.values.get(0), "S");
+
+ getMapValueFromKey.execute(tuple2, collector);
+ assertEquals(collector.values.get(0), "T");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetTimeTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetTimeTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetTimeTest.java
new file mode 100644
index 0000000..57dc090
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetTimeTest.java
@@ -0,0 +1,34 @@
+package com.jwplayer.sqe.trident.function;
+
+import com.jwplayer.sqe.trident.SingleValuesCollector;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+import java.util.Date;
+
+
+import static org.junit.Assert.assertEquals;
+
+public class GetTimeTest {
+ @Test
+ public void testGetTimeFromDate() {
+ GetTime getTime = new GetTime();
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("Date"), new Date(0l));
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ getTime.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 0l);
+ }
+
+ @Test
+ public void testGetTimeFromString() {
+ GetTime getTime = new GetTime();
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("Date", "DateFormat"), "1970-01-01 00:00:00", "yyyy-MM-dd HH:mm:ss");
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ getTime.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 0l);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/HashTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/HashTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/HashTest.java
new file mode 100644
index 0000000..98ff587
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/HashTest.java
@@ -0,0 +1,24 @@
+package com.jwplayer.sqe.trident.function;
+
+import com.google.common.primitives.Longs;
+import com.jwplayer.sqe.trident.SingleValuesCollector;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+import static org.junit.Assert.*;
+
+
+public class HashTest {
+ @Test
+ public void testMurmur2Hash() {
+ Hash hash = new Hash();
+ SingleValuesCollector collector = new SingleValuesCollector();
+ TridentTuple tuple =
+ TridentTupleView.createFreshTuple(new Fields("HashName", "ValueToHash"), "Murmur2", "ABCDEF");
+
+ hash.execute(tuple, collector);
+ assertEquals(Longs.fromByteArray((byte[]) collector.values.get(0)), 0xF6106105304A543l);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/IsInTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/IsInTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/IsInTest.java
new file mode 100644
index 0000000..aecb919
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/IsInTest.java
@@ -0,0 +1,40 @@
+package com.jwplayer.sqe.trident.function;
+
+import com.jwplayer.sqe.trident.SingleValuesCollector;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+import static org.junit.Assert.*;
+
+
+public class IsInTest {
+ @Test
+ public void testIsInNumbers() {
+ IsIn isIn = new IsIn();
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("A", "1", "2"), 1.0f, 1.1d, 2l);
+ isIn.execute(tuple, collector);
+ assertFalse((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("A", "1", "2", "3"), 1.0f, 1.1d, 2l, 1);
+ isIn.execute(tuple, collector);
+ assertTrue((Boolean) collector.values.get(0));
+ }
+
+ @Test
+ public void testIsInString() {
+ IsIn isIn = new IsIn();
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("A", "1", "2"), "A", "cat", "moo");
+ isIn.execute(tuple, collector);
+ assertFalse((Boolean) collector.values.get(0));
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("A", "1", "2", "3"), "A", "cat", "A", "moo");
+ isIn.execute(tuple, collector);
+ assertTrue((Boolean) collector.values.get(0));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ParseDateTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ParseDateTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ParseDateTest.java
new file mode 100644
index 0000000..9f8b351
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ParseDateTest.java
@@ -0,0 +1,27 @@
+package com.jwplayer.sqe.trident.function;
+
+import static org.junit.Assert.*;
+
+import com.jwplayer.sqe.trident.SingleValuesCollector;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+import java.util.Date;
+
+
+public class ParseDateTest {
+ @Test
+ public void testParseDate() {
+ ParseDate parseDate = new ParseDate();
+ String date = "1970-01-01 00:00:00";
+ String format = "yyyy-MM-dd HH:mm:ss";
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("Date", "Format"), date, format);
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ parseDate.execute(tuple, collector);
+
+ assertEquals(collector.values.get(0), new Date(0l));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ProcessArithmeticOperatorTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ProcessArithmeticOperatorTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ProcessArithmeticOperatorTest.java
new file mode 100644
index 0000000..c480dea
--- /dev/null
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ProcessArithmeticOperatorTest.java
@@ -0,0 +1,255 @@
+package com.jwplayer.sqe.trident.function;
+
+import static org.junit.Assert.*;
+
+import com.jwplayer.sqe.language.expression.transform.ArithmeticOperatorType;
+import com.jwplayer.sqe.trident.SingleValuesCollector;
+import org.junit.Test;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.trident.tuple.TridentTupleView;
+import org.apache.storm.tuple.Fields;
+
+
+public class ProcessArithmeticOperatorTest {
+ @Test
+ public void testAddition() {
+ ProcessArithmeticOperator operator = new ProcessArithmeticOperator(ArithmeticOperatorType.Addition);
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2, 2);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 4l);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2, 2l);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 4l);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2, 2.0f);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 4.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2, 2.0d);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 4.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2l, 2l);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 4l);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2l, 2.0f);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 4.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2l, 2.0d);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 4.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2.0f, 2.0f);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 4.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2.0f, 2.0d);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 4.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2.0d, 2.0d);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 4.0d);
+ }
+
+ @Test
+ public void testDivision() {
+ ProcessArithmeticOperator operator = new ProcessArithmeticOperator(ArithmeticOperatorType.Division);
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 4, 2);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 2l);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 5, 2);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 2.5d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 4, 2l);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 2l);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 5, 2l);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 2.5d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 5, 2.0f);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 2.5d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 5, 2.0d);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 2.5d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 4l, 2l);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 2l);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 5l, 2l);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 2.5d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 5l, 2.0f);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 2.5d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 5l, 2.0d);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 2.5d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 5.0f, 2.0f);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 2.5d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 5.0f, 2.0d);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 2.5d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 5.0d, 2.0d);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 2.5d);
+ }
+
+ @Test
+ public void testModulus() {
+ ProcessArithmeticOperator operator = new ProcessArithmeticOperator(ArithmeticOperatorType.Modulus);
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 5, 2);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 1l);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 5, 2l);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 1l);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 5, 2.0f);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 1.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 5, 2.0d);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 1.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 5l, 2l);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 1l);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 5l, 2.0f);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 1.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 5l, 2.0d);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 1.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 5.0f, 2.0f);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 1.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 5.0f, 2.0d);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 1.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 5.0d, 2.0d);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 1.0d);
+ }
+
+ @Test
+ public void testMultiplication() {
+ ProcessArithmeticOperator operator = new ProcessArithmeticOperator(ArithmeticOperatorType.Multiplication);
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2, 2);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 4l);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2, 2l);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 4l);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2, 2.0f);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 4.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2, 2.0d);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 4.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2l, 2l);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 4l);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2l, 2.0f);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 4.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2l, 2.0d);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 4.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2.0f, 2.0f);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 4.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2.0f, 2.0d);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 4.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2.0d, 2.0d);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 4.0d);
+ }
+
+ @Test
+ public void testSubtraction() {
+ ProcessArithmeticOperator operator = new ProcessArithmeticOperator(ArithmeticOperatorType.Subtraction);
+ SingleValuesCollector collector = new SingleValuesCollector();
+
+ TridentTuple tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2, 2);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 0l);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2, 2l);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 0l);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2, 2.0f);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 0.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2, 2.0d);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 0.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2l, 2l);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 0l);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2l, 2.0f);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 0.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2l, 2.0d);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 0.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2.0f, 2.0f);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 0.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2.0f, 2.0d);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 0.0d);
+
+ tuple = TridentTupleView.createFreshTuple(new Fields("num1", "num2"), 2.0d, 2.0d);
+ operator.execute(tuple, collector);
+ assertEquals(collector.values.get(0), 0.0d);
+ }
+}
[10/11] storm git commit: SQE: Import wiki documentation
Posted by pt...@apache.org.
SQE: Import wiki documentation
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/7409c91f
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/7409c91f
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/7409c91f
Branch: refs/heads/sqe_import
Commit: 7409c91ffa1c7ae181357d36d130ccf6d10b765f
Parents: b28921d
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Wed Sep 28 16:15:52 2016 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Wed Sep 28 16:15:52 2016 -0400
----------------------------------------------------------------------
docs/storm-sqe-commands.md | 160 ++++++++++++++++++++++++++++++++
docs/storm-sqe-expressions.md | 59 ++++++++++++
docs/storm-sqe-options.md | 20 ++++
docs/storm-sqe-replay-filtering.md | 33 +++++++
docs/storm-sqe-states.md | 123 ++++++++++++++++++++++++
docs/storm-sqe-streams.md | 104 +++++++++++++++++++++
docs/storm-sqe-using.md | 15 +++
docs/storm-sqe.md | 28 ++++++
8 files changed, 542 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/7409c91f/docs/storm-sqe-commands.md
----------------------------------------------------------------------
diff --git a/docs/storm-sqe-commands.md b/docs/storm-sqe-commands.md
new file mode 100644
index 0000000..65c589e
--- /dev/null
+++ b/docs/storm-sqe-commands.md
@@ -0,0 +1,160 @@
+# Commands
+
+SQE currently supports the following commands:
+
+* CreateStream
+* Query
+* Set
+
+All commands are formatted as a JSON map with a single entry. The key is the name of the command and the value is a map formatted according to the command type. This command map can contain multiple fields, sections or clauses. Commands are typically loaded from one or more JSON files that are an array of commands:
+
+ [
+ {"Set": {...}},
+ {"Set": {...}},
+ {"CreateStream": {...}},
+ {"Query": {...}},
+ {"Query": {...}}
+ ]
+
+## CreateStream
+
+CreateStream allows for creating input streams to the topology. For example, you can create a stream from a topic in a Kafka cluster. The JSON representation looks like:
+
+ {
+ "createStream": {
+ "streamName": "<STREAM_NAME>",
+ "objectName": "<OBJECT_NAME>",
+ "spoutName": "<SPOUT_NAME>",
+ "spoutType": "<SPOUT_TYPE>",
+ "deserializer": "<DESERIALIZER>",
+ "options": {
+ "option1": value1,
+ "optionN": valueN
+ }
+ }
+ }
+
+* streamName - This is the name of the stream used by SQE. Queries reference streams by using this name in the FROM clause.
+* objectName - This is the name of the object to read from. For example, with Kafka, this would be the topic. Different spouts will handle this differently, but it's always a reference to some division of data from the data source.
+* spoutName - This is the name of the spout to use. Currently, Kafka and Fixed are supported.
+* spoutType (optional) - This determines the type of spout, which corresponds to the state types (NON_TRANSACTIONAL, TRANSACTIONAL, OPAQUE). Exactly once processing guarantees are determined by the combination of spout type and state type. Some spouts require this, while others do not. Not all types are supported by all spouts. (see Trident documentation for more information)
+* deserializer (optional) - This takes the output of the spout and deserializes it. The only currently supported deserializer is Avro, which takes a byte array, converts it into an Avro record, then adds the required fields (based on supplied queries) to the tuple. If no deserializer is specified, then the tuples emitted by the spout are unchanged. For example, streams from a Kafka spout will typically need a deserializer since data is stored as a single message. Other spouts may return data split into different fields that can be queried against directly.
+* options (optional) - This object allows you to provide command level spout options. It can include new options that affect how the spout emits data or overrides for global SQE options that are passed to the spout. For example, you can specify the hostname of the spout you are reading from.
+
+### Example
+
+ {
+ "CreateStream": {
+ "streamName": "big.query.data",
+ "objectName": "big.query.data",
+ "spoutName": "FIXED",
+ "spoutType": "NON_TRANSACTIONAL",
+ "options": {
+ "jw.sqe.spout.fixed.fields": ["DateString", "AccountToken", "UserName", "HappyDanceCount"],
+ "jw.sqe.spout.fixed.values": [
+ ["2015-05-01 00:00", "Account1", "Joe", 1],
+ ["2015-05-01 01:00", "Account1", "Bob", -1],
+ ["2015-05-01 02:00", "Account1", "Susy", 1],
+ ["2015-05-01 03:00", "Account1", "Mr. Fancy Pants", 1],
+ ["2015-05-01 04:00", "Account1", "Joe", -1],
+ ["2015-05-01 05:00", "Account1", "Bob", 1],
+ ["2015-05-01 06:00", "Account1", "Susy", 1],
+ ["2015-05-01 07:00", "Account1", "Mr. Fancy Pants", 1],
+ ["2015-05-01 08:00", "Account2", "Joe", 1],
+ ["2015-05-01 09:00", "Account2", "Bob", 1],
+ ["2015-05-01 10:00", "Account2", "Susy", -1],
+ ["2015-05-01 11:00", "Account2", "Mr. Fancy Pants", 1],
+ ["2015-05-01 12:00", "Account2", "Joe", 1],
+ ["2015-05-01 13:00", "Account2", "Bob", 1],
+ ["2015-05-01 14:00", "Account2", "Susy", 1],
+ ["2015-05-01 15:00", "Account2", "Mr. Fancy Pants", -1]
+ ]
+ }
+ }
+ }
+
+## Set
+
+Set adds or overwrites entries in the global config map. This can be used to set config options within a command file. The JSON representation looks lke:
+
+ {"Set": {"key":"<KEY>","value":"<VALUE>"}}
+
+### Example
+
+ {"Set": {"key":"jw.sqe.state.redis.datatype","value":"HASH"}}
+
+## Query
+
+Queries allow you to run SQL like queries against an input stream and to persist/aggregate against a Trident state. The JSON representation of a query looks like:
+
+ {
+ "Query": {
+ "insertInto":{
+ "objectName":"<OBJECT_NAME>",
+ "stateName": "<STATE_NAME>",
+ "stateType": "<STATE_TYPE>",
+ "fields":[<FIELD_LIST>],
+ "options": {
+ "option1": value1,
+ "optionN": valueN
+ }
+ },
+ "select":{"expressions":[<EXPRESSION_LIST>]},
+ "from":{"objectName":"<OBJECT_NAME>"},
+ "where":<PREDICATE_EXPRESSION>
+ }
+ }
+
+### InsertInto
+
+This clause tells the query engine what object (table, view, stream, etc.) the results are delivered to, what the fields are named, and if it needs to persist data to a certain state. Similar to SQL, the ordering of the fields in this clause lines up with the expressions in the Select clause. InsertInto contains the following fields:
+
+* objectName - If SQE is sending the results to an output stream, then this is the name of the stream in the output stream map returned by build(). If a state is supplied (see below), then this is the name of table (or other object) the results are persisted to.
+* stateName (optional) - If this field is supplied, then SQE will persist the results to a data store using this state. The name here must be a supported state, such as Redis. Otherwise, the results are streamed through the output stream supplied by build().
+* stateType (optional) - This determines the type of state (NON_TRANSACTIONAL, TRANSACTIONAL, OPAQUE). It must be supplied if stateName is.
+* fields - When persisting to a state, these fields are the names of the keys or columns. When using an output stream, they are the names of the fields in each tuple. The fields here are ordered and line up with the expressions in the Select clause.
+* options (optional) - This object allows you to provide query level state options. It can include new options that affect how the state persists data or overrides for global SQE options that are passed to the state. For example, you can specify the hostname of the data store you are persisting data to.
+
+### Select
+
+This clause represents the selection of fields and transformations and aggregations on those fields from the input streams. The only field of the Select clause is a list of expressions. Expressions can come in one of three types:
+
+* Field - Field expressions directly reference fields in the input streams and are represented as a string literal of the field name in JSON
+* Constant - A constant value that can be a number, string, boolean, or null. The JSON representation looks like: `{"C":<CONSTANT>}`. Non-string constants can also be represented as just the literal. Internally, numerical constants are represented as either an Integer, Long or Double value.
+* Function - Functions are expressions that represent processing of input data. There are two kinds of function expressions: transform and aggregation. There is also a special type of transform function called a predicate expression that always evaluates to true/false. Functions contain an argument list of expressions, which can include fields, constants and transform expressions. Aggregation expressions cannot be contained in another expression. Function expressions are represented in JSON as: `{"<FUNCTION_NAME>":[<EXPRESSION_LIST>]}`.
+
+### From
+
+The From clause tells SQE what input stream the query is referencing. The only field is "objectName." Typically, this is going to be the name of the Avro schema of the input data, but can include named input streams of tuples in the future.
+
+### Where
+
+The Where clause is an optional clause that allows filtering of data in the query by specifying which data should be kept. This clause accepts a single predicate expression, but you can chain together expressions using expressions like AND and OR, similar to other languages.
+
+### No Group By?
+
+SQE automatically determines which expressions are key fields and which expressions are aggregate/value fields. If a top level expression in the Select clause is a field, constant or transform expression, then it is a key field for the purposes of aggregations and persisting state. If a top level expression is an aggregation expression, then it is a value field.
+
+### Example
+
+This is an example of a query of top level device analytics by the minute. It includes sums for embeds, plays, completes. Additionally, it creates a HyperLogLog bitmap of all UserTokens for play events. The input stream is filtered to only include data has positive embeds, plays, or completes. It then uses the transactional RedisMapState to persist the results.
+
+ {
+ "Query": {
+ "insertInto":{
+ "objectName":"MinuteDeviceMeasures",
+ "stateName": "REDIS",
+ "stateType": "TRANSACTIONAL",
+ "fields":["DateTime","Device","Embeds","Plays","Completes","TimeWatched","AdImpressions","HllViewers"]
+ },
+ "select":{
+ "expressions":[
+ {"FormatDate":[{"ParseDate":["DateGMTISO",{"C":"yyyy-MM-dd HH:mm:ss Z"}]},{"C":"yyyy-MM-dd'T'HH"}]}, "Device",
+ {"Sum":["Embeds"]},{"Sum":["Plays"]},{"Sum":["Completes"]},{"Sum":["TimeWatched"]},{"Sum":["AdImpressions"]},
+ {"CreateHll":[{"If": [{"GreaterThan":["Plays",0]}, "UserToken",null]}]}
+ ]
+ },
+ "from":{"objectName":"com.jwplayer.analytics.avro.Ping"},
+ "where":{"Or":[{"Or":[{">":["Embeds",0]},{">":["Plays",0]}]},{">":["Completes",0]}]}
+ }
+ }
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/7409c91f/docs/storm-sqe-expressions.md
----------------------------------------------------------------------
diff --git a/docs/storm-sqe-expressions.md b/docs/storm-sqe-expressions.md
new file mode 100644
index 0000000..38bc2d5
--- /dev/null
+++ b/docs/storm-sqe-expressions.md
@@ -0,0 +1,59 @@
+# Supported Expressions
+
+## Aggregate
+
+All aggregate expressions take a single argument and output a single value.
+
+* CreateHll - Creates a bitmap representing a HyperLogLog object using the Clearspring implementation
+* CreateHllp - Similar to CreateHll, but uses the HyperLogLogPlus algorithm
+* Max - Returns the maximum value in a set of objects. If the input values are numbers, clojure.lang.Numbers.max() is used, otherwise Java's Comparable interface is used to perform comparisons. Objects that do not implement Comparable are not supported.
+* Sum - Sums a set of numbers into a single value. Works similarly to Sum in SQL. Uses Trident's built in Sum aggregator.
+
+## Transform
+
+Conditionals
+
+* If - `[<PREDICATE>,<TRUE_VALUE>,<FALSE_VALUE>]` - Returns \<TRUE_VALUE\> if \<PREDICATE\> is true, otherwise returns \<FALSE_VALUE\>
+
+Date/Time
+
+* FormatDate - `[<DATE>,<OUTPUT_DATE_FORMAT>]` - Takes \<DATE\> and formats it into a string using \<OUTPUT_DATE_FORMAT\>. Uses Java's SimpleDateFormat and the date is adjusted to UTC.
+* GetTime - `[<DATE>]` - Turns a date object into a Long timestamp using the getTime method.
+* ParseDate - `[<DATE_STRING>,<INPUT_DATE_FORMAT>]` - Takes \<DATE_STRING\> and parses it into a Date object using \<OUTPUT_DATE_FORMAT\>. Uses Java's SimpleDateFormat and the date is adjusted to UTC.
+* RoundDate - `[<DATE>,<AMOUNT>,<UNIT>]` - Takes a date object and rounds it to the nearest \<AMOUNT\> \<UNIT\>. For example, if \<AMOUNT\> = 10 and \<UNIT\> = Second, then it will round to the nearest 10th second. Allowed units are Second, Minute, Hour, and Day.
+
+Math
+
+* + - Addition
+* / - Division
+* % - Modulus
+* * - Multiplication
+* - - Subtraction
+
+Predicate
+
+* = - `[<VALUE_A>,<VALUE_B>]` - Compares \<VALUE_A\> to \<VALUE_B\>. If both values are Numbers, clojure.lang.Numbers.equiv is used to perform the comparison. Otherwise, the equals method of \<VALUE_A\> is called.
+* In - Returns true if the first argument is equal to one of the remaining arguments. Uses clojure.lang.Numbers.equiv for Number objects, or the equals method for other object types. Similar to the SQL keyword of the same name.
+* Logical Operators - And, Not, Or, Xor - Each takes 2 or more arguments, except for Not, which only takes one. If more than 2 arguments is supplied for And/Or/Xor, the operator is chained, i.e. `{"And":[A,B,C,D]}` is the same as `{"And":[{"And":[{"And":[A,B]},C]},D]}` is the same as `A and B and C and D`).
+* Numerical Comparators - >, >=, <, <=, != - Each takes 2 arguments. Uses clojure.lang.Numbers to perform the comparisons.
+* RLike - `[<STRING>,<REG_EX>]` - Similar to RLike in MySQL or Hive. Uses the String method matches to evaluate the regular expression. The function caches compilations of the `<REG_EX>` values it sees, though typically that expression is a constant.
+
+Map
+
+* ExpandKeys - Takes a Java Map object and emits each key. Unlike most other transformation expressions which add a single value onto a tuple, this takes an input tuple and creates a new tuple for each key.
+* ExpandValues - Takes a Java Map object and emits each value. Similar to ExpandKeys, this can create multiple tuples for each input tuple.
+
+Other
+
+* Hash - `[<ALGORITHM_NAME>,<VALUE>]` - Hashes the value using the given algorithm. The hash is returned as a byte array. Supported algorithms:
+ * Murmur2 - Uses Clearspring's Murmur2 implementation. The output is always 8 bytes.
+
+## Adding new expressions
+
+Adding a new expressions requires the following changes:
+
+* Sub-classing a new expression from the appropriate parent class (AggregationExpression, TransformExpression, or PredicateExpression) and overriding the abstract methods. These classes allow the query engine to parse and operate on queries and to interact with the input streams.
+* Updating the /META-INF/services/com.jwplayer.sqe.language.expression.FunctionExpression resource file to include the expressions class. You can also add "UDFs" to a project that includes SQE as a dependency by adding a new /META-INF/services/com.jwplayer.sqe.language.expression.FunctionExpression resource file in that project. Expression names used in queries are pulled from the getFunctionName method in each expression's class. These names are case insensitive to the parser. Because SQE includes a version of this file, you'll need to use something like Maven's Shade plugin (https://maven.apache.org/plugins/maven-shade-plugin/examples/resource-transformers.html#AppendingTransformer) to combine the original resource file, with your custom resource file to combine the base SQE expressions and your UDFs.
+* Creating new classes from the Trident BaseAggregator, CombinerAggregator, ReducerAggregator or BaseFunction classes. These classes operate directly on the stream. One or more expressions will parse to one of the supported expression types, which in turn interface with one or more Trident Aggregators or Functions.
+
+When writing new expressions, it shouldn't be important to know what types of expressions the arguments are. Field expressions should be handled naturally because the values of that field will come in through each tuple you process. Constant and Transform expressions that are nested in the argument list will appear in tuples the same as a field. Behind the scenes, SQE processes nested transform expressions first. By the time a transform function higher up in an expression tree is processed, its transform function arguments already appear as fields in the tuple.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/7409c91f/docs/storm-sqe-options.md
----------------------------------------------------------------------
diff --git a/docs/storm-sqe-options.md b/docs/storm-sqe-options.md
new file mode 100644
index 0000000..3c917a0
--- /dev/null
+++ b/docs/storm-sqe-options.md
@@ -0,0 +1,20 @@
+# Options
+
+## Topology Config
+
+The config file for the SQE Topology class allows you to specify both Storm and SQE options using a YAML file. The format of this file looks like:
+
+ Storm:
+ topology.workers: 2
+ other.storm.option: value
+ SQE:
+ jw.sqe.parallelism.hint: 10
+ jw.sqe.spout.kafka.zkhosts:
+ - zookeeper:2181
+ other.sqe.option: value
+
+The key/value pairs in the Storm section are added to the Topology config. Pairs in the SQE section are added to the SQE config. Options specified in a command file (either through the SET command or within an options section) override options specified in the config file.
+
+## Query Engine
+
+* jw.sqe.parallelism.hint - This determines the parallelism for each input stream. So a paralellism hint of 20 with an input stream plus the primary processing bolt would translate into 40 executors. This does not affect the number of executors for downstream global aggregations or bolts used for persisting state. These bolts currently get a single executor each.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/7409c91f/docs/storm-sqe-replay-filtering.md
----------------------------------------------------------------------
diff --git a/docs/storm-sqe-replay-filtering.md b/docs/storm-sqe-replay-filtering.md
new file mode 100644
index 0000000..8e150b0
--- /dev/null
+++ b/docs/storm-sqe-replay-filtering.md
@@ -0,0 +1,33 @@
+# Replay Filtering
+
+Replay filtering in SQE deals with removing replays using a lightweight mechanism from a data source that can't be written to using the standard transactional or opaque semantics. Currently, this means writing to Kafka and later removing replays while reading from the data source in downstream topologies, though the solution here can potentially be used with other data stores, streams and states. The solution to this problem is inspired by the Kafka idempotent producer proposal [here](https://cwiki.apache.org/confluence/display/KAFKA/Idempotent+Producer).
+
+## How it works
+
+You can read a more detailed explanation in the link above, but the idea is pretty simple. Let's say you have two topologies. The first reads from topic 1 and writes to topic 2. The second reads from topic 2 and writes to a transactional data store. The first topology can replay records and write them into Kafka multiple times. For example, this can happen if a batch failure happens.
+
+What we want to do is ignore those replays in the second topology. We can do this if the first topology meets the following criteria:
+
+* Tuples are read and processed in the order they appear in the data source. For Kafka, this means reading and processing the records in offset order. If the spout emits a batch of data and emits it in a random order, then replays cannot be removed using this method.
+* Partitioning is deterministic and maintained between the data source, inside Storm, and in the data store. This means no re-partitioning of streams inside Storm. We include a special partitioner that determines the destination partition based on the source partition. If the destination topic has the same (or more) partitions, source partition should equal destination partition. If the destination has fewer partitions, then: destination partition = source partition % total # of destination partitions.
+
+With the above, we are able to filter replayed messages by tracking each "highwater mark (offset)" per partition. We also include a PID that identifies the producer and source, which can be from a SQE topology or other producer. This allows us to track different highwater marks for different producers and sources. If we see a record with an offset that is below the current highwater mark for it's PID and partition, we know we've already processed that record and we can toss the replay. If instead the highwater mark is higher, then we emit it normally and update the highwater mark.
+
+To track the needed information, SQE adds a stream metadata object to each tuple. This metadata includes a PID (represented as a long), a partition (integer), and a message offset (long). For a Kafka stream, the PID is generated by hashing the topology name, stream name, ZK hosts, and ZK path together. Even though partition and offset map directly to the same concepts in Kafka, it can be extended to other streams/spouts. For storage in Kafka, the stream metadata is converted to a size 20 byte array with PID, partition, offset order.
+
+In the future, we may be able to extend this to (partition) local aggregations, as long as we can produce a single piece of metadata from multiple tuples (group on PID and partition, max on offset?) and maintain offset ordering for each aggregation batch. This is not supported at the moment, however.
+
+## How to enable replay filtering
+
+### State Options
+
+Set the following options for the Kafka state to record the stream metadata and to ensure records are written to the appropriate partitions:
+
+* jw.sqe.state.kafka.keytype - StreamMetadata
+* jw.sqe.state.kafka.partitionClass - com.jwplayer.sqe.language.state.kafka.SourcePartitionPartitioner
+
+### Stream Options
+
+If the stream metadata is included in the key of each record (records with NULL or invalid keys don't get filtered), all that needs to be done on the stream is to enable replay filtering:
+
+* jw.sqe.spout.kafka.filterReplays - true
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/7409c91f/docs/storm-sqe-states.md
----------------------------------------------------------------------
diff --git a/docs/storm-sqe-states.md b/docs/storm-sqe-states.md
new file mode 100644
index 0000000..c620f04
--- /dev/null
+++ b/docs/storm-sqe-states.md
@@ -0,0 +1,123 @@
+# States
+
+## Kafka
+
+SQE uses a slightly modified version of the storm-kafka project for its Kafka state implementation. The objectName in the InsertInto clause is the topic that messages are written to. Depending on the particular key type that is used, this state accept one or two fields which correspond to the key and value written to the Kafka topic.
+
+### Options
+
+Most, but not all, of these options map to the underlying Kafka producer options.
+
+* jw.sqe.state.kafka.brokers - An array of Kafka brokers. Maps to metadata.broker.list as a comma delimited string.
+* jw.sqe.state.kafka.serializerClass (default: org.apache.kafka.common.serialization.ByteArraySerializer) - Maps to value.serializer
+* jw.sqe.state.kafka.key.serializerClass (default: org.apache.kafka.common.serialization.ByteArraySerializer) - Maps to key.serializer
+* jw.sqe.state.kafka.partitionClass (default: org.apache.kafka.clients.producer.internals.DefaultPartitioner) - Maps to partitioner.class
+* jw.sqe.state.kafka.producerType (default: async) - Maps to producer.type
+* jw.sqe.state.kafka.request.requiredAcks (default: 1) - Maps to acks
+* jw.sqe.state.kafka.keytype (default: MessageHash) - This determines how the key for each Kafka record is created. The three options are:
+ * Field - Normally, only one field is allowed, which is written as the value, and the key is generated. If the field key type is used, then two fields are required. The first is the key and the second is the value.
+ * MessageHash - Generates the key by hashing (using Murmur2) the value
+ * StreamMetadata - Generates a 20 byte key from the internal metadata SQE attaches to each tuple. This is useful for lightweight replay filtering in downstream SQE topologies. See [Replay Filtering](https://github.com/jwplayer/sqe/wiki/replay-filtering) for more information.
+
+### Example
+
+ "insertInto": {
+ "objectName":"ping-avro-sqe-all-pings-test",
+ "stateName": "Kafka",
+ "stateType": "NON_TRANSACTIONAL",
+ "fields" : ["message"],
+ "jw.sqe.state.kafka.keytype": "StreamMetadata",
+ "jw.sqe.state.kafka.producerType": "sync",
+ "jw.sqe.state.kafka.partitionClass": "com.jwplayer.sqe.language.state.kafka.SourcePartitionPartitioner",
+ "jw.sqe.state.kafka.brokers": ["mykafka-01", "mykafka-02"]
+ }
+
+## MongoDB
+
+SQE uses an internal map state implementation to persist data to Mongo DB. The objectName in the InsertInto clause is the collection to write to and the fields are the fields names in each record. One record is written per value field for each set of keys. The _id field is generated from a hash of the keys plus the value field name written into a byte array. The order of the keys here matches the order of the key field names in the InsertInto clause. For transactional and opaque states, value fields are written as sub-documents containing the appropriate transaction ID and other fields.
+
+### Options
+
+* jw.sqe.state.mongodb.cachesize (default: 5000) - The maximum number of objects in the map cache
+* jw.sqe.state.mongodb.db - The database to write to
+* jw.sqe.state.mongodb.hosts - A list of hostnames to connect to
+* jw.sqe.state.mongodb.password - The password
+* jw.sqe.state.mongodb.port (default: 27017) - The port
+* jw.sqe.state.mongodb.replicaSet - The replica set name
+* jw.sqe.state.mongodb.userName - The username
+
+### Example
+
+ "insertInto":{
+ "objectName":"DailyAcctEPCUniq",
+ "stateName": "MONGO",
+ "stateType": "TRANSACTIONAL",
+ "fields":["date","accounttoken","ttlmarker","uniques","embeds","plays","completes","tw"]
+ "options": {
+ "jw.sqe.state.mongodb.db": "mydb",
+ "jw.sqe.state.mongodb.hosts": ["mymongodb.com"],
+ "jw.sqe.state.mongodb.password": "password",
+ "jw.sqe.state.mongodb.replicaSet": "myreplicaset",
+ "jw.sqe.state.mongodb.userName": "bob"
+ }
+ }
+
+## Redis
+
+SQE uses a modified version of the storm-redis project for its Redis map state. Query data can be persisted as Redis strings or hashes, with string being the default. SQE uses a custom serializer using Gson. For transactional states, the format is in JSON: `[<TXID>,<VALUE_TYPE>,<VALUE>]`. Opaque states are formatted like: `[<TXID>,<VALUE_TYPE>,<VALUE>]`. Currently supported value types are:
+
+* N - null
+* I - Integer
+* L - Long
+* F - Float
+* D - Double
+* S - String
+* B - Byte Array
+
+An example transactional value is: `[6491,"I",0]`
+
+How the InsertInto fields are translated into Redis data structures depends on the Redis data type being used. The two supported types are string and hash.
+
+### String
+
+String is the default data type used by SQE. A custom KeyFactory is used that formats the aggregated data like: `<OBJECT_NAME>:<KEY_FIELD1>:...:<KEY_FIELDN>:<VALUE_FIELD>`. ':' is the default delimiter, but it can be configured (see the Options section below). For example, if you have the key field "Timestamp" and value field "AdImpressions" in your InsertInto clause, a key/value pair in Redis could look like:
+
+Key: `HourlyEvents:2015-05-18T20:AdImpressions`
+Value: `[1234,"I",5678]`
+
+This is the simplest way to send data to Redis. However, for large amounts of data, it's often better to format the data using the hash data type.
+
+### Hash
+
+This data type splits the InsertInto key fields between the hash's key name and field name based on the "jw.sqe.state.redis.keyname.fields" and "jw.sqe.state.redis.fieldname.fields" options in the query (see the Options section below). For example, if you have the following InsertInto clause in your query:
+
+ "insertInto":{
+ "objectName":"TestQuery",
+ "stateName": "REDIS",
+ "stateType": "TRANSACTIONAL",
+ "fields":["Account","Timestamp","Domain","Views","Clicks"],
+ "options": {
+ "jw.sqe.state.redis.datatype": "HASH",
+ "jw.sqe.state.redis.keyname.fields": ["Account","Timestamp"],
+ "jw.sqe.state.redis.fieldname.fields": ["Domain"]
+ }
+ }
+
+The data in Redis would look like:
+
+Key Name: `TestQuery:ABCD-1234:2015-05-18T20:`
+Field Name: `bob.com:Views`
+Value: `[1234,"I",5678]`
+
+Another way to think of this, is that you are splitting the standard Redis string representation in two. The first half is used for the Redis key name, the second for the Redis field name. The objectName always appears first in the Redis key name and the value field name always appears at the end of the Redis field name. The same delimiter option is used as for Redis strings.
+
+### Options
+
+* jw.sqe.state.redis.database (default: 0) - The Redis DB to write to
+* jw.sqe.state.redis.datatype (default: STRING) - The Redis data type to use for storing the queried data in Redis. The currently available options are STRING and HASH. When using the HASH data type, you should also include the "jw.sqe.state.redis.fieldname.fields" and "jw.sqe.state.redis.keyname.fields" options so the RedisMapState knows how to build the key names and field names from the given keys from the query.
+* jw.sqe.state.redis.delimiter (default: :) - The delimiter used by the KeyFactory
+* jw.sqe.state.redis.expireintervalsec (default: 0) - Sets the expiration TTL for all keys. No TTL is set if this is 0.
+* jw.sqe.state.redis.fieldname.fields - A list of key fields used to create the field name in a Redis hash
+* jw.sqe.state.redis.host - The host of the Redis server
+* jw.sqe.state.redis.keyname.fields - A list of key fields used to create the key name in a Redis hash
+* jw.sqe.state.redis.port (default: 6379) - The port of the Redis server
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/7409c91f/docs/storm-sqe-streams.md
----------------------------------------------------------------------
diff --git a/docs/storm-sqe-streams.md b/docs/storm-sqe-streams.md
new file mode 100644
index 0000000..8d39dfa
--- /dev/null
+++ b/docs/storm-sqe-streams.md
@@ -0,0 +1,104 @@
+# Streams
+
+Streams are ways of accessing data from data stores as streams of tuples. Some spouts emit data that may not be immediately useful to a SQE query. For example, Kafka writes tuples that contain a message field. These messages may be serialized data or otherwise contain complex information and objects you want to access. Deserializers allow us to access this data and pull fields out of them that can then be queried against. See more information about deserializers below.
+
+## Fixed
+
+SQE uses the FixedBatchSpout provided by Storm for testing. This allows you to write a list of of fields and values that will be emitted onto the stream as part of a query.
+
+### Options
+
+* jw.sqe.spout.fixed.fields - The fields that are emitted onto the stream as an array
+* jw.sqe.spout.fixed.values - An array of value arrays that the spout emits onto the stream. Each array is emitted as a tuple and the order of values in each array corresponds to the order of the fields.
+
+### Example
+
+ {
+ "CreateStream": {
+ "streamName": "big.query.data",
+ "objectName": "big.query.data",
+ "spoutName": "FIXED",
+ "spoutType": "NON_TRANSACTIONAL",
+ "options": {
+ "jw.sqe.spout.fixed.fields": ["DateString", "AccountToken", "UserName", "HappyDanceCount"],
+ "jw.sqe.spout.fixed.values": [
+ ["2015-05-01 00:00", "Account1", "Joe", 1],
+ ["2015-05-01 01:00", "Account1", "Bob", -1],
+ ["2015-05-01 02:00", "Account1", "Susy", 1],
+ ["2015-05-01 03:00", "Account1", "Mr. Fancy Pants", 1],
+ ["2015-05-01 04:00", "Account1", "Joe", -1],
+ ["2015-05-01 05:00", "Account1", "Bob", 1],
+ ["2015-05-01 06:00", "Account1", "Susy", 1],
+ ["2015-05-01 07:00", "Account1", "Mr. Fancy Pants", 1],
+ ["2015-05-01 08:00", "Account2", "Joe", 1],
+ ["2015-05-01 09:00", "Account2", "Bob", 1],
+ ["2015-05-01 10:00", "Account2", "Susy", -1],
+ ["2015-05-01 11:00", "Account2", "Mr. Fancy Pants", 1],
+ ["2015-05-01 12:00", "Account2", "Joe", 1],
+ ["2015-05-01 13:00", "Account2", "Bob", 1],
+ ["2015-05-01 14:00", "Account2", "Susy", 1],
+ ["2015-05-01 15:00", "Account2", "Mr. Fancy Pants", -1]
+ ]
+ }
+ }
+ }
+
+## Kafka
+
+The Kafka stream type allows you to read data from a Kafka topic onto a stream. Typically, though not necessarily, you will use a deserializer on the message to create the appropriate fields on the stream. The object name in the CreateStream command is the topic you are reading from. By default, without using a deserializer, the key and value are outputted under the fields _key and _value, respectively.
+
+### Options
+
+* jw.sqe.spout.kafka.zkhosts - An array of Zookeeper hosts, including the port. This is used to locate the Kafka cluster.
+* jw.sqe.spout.kafka.clientid - The client ID used by the Kafka spout
+* jw.sqe.spout.kafka.bufferSizeBytes (optional)
+* jw.sqe.spout.kafka.fetchSizeBytes (optional)
+* jw.sqe.spout.kafka.maxOffsetBehind (optional)
+* jw.sqe.spout.kafka.filterReplays (default: false) - Enables replay filtering based on stream metadata recorded in the key of each record. See [Replay Filtering](https://github.com/jwplayer/sqe/wiki/replay-filtering) for more information.
+* jw.sqe.spout.kafka.filterReplays.metadata.ttl (default 172800) - The TTL (in seconds) of any individual metadata/highwater marks recorded by the replay filtering functionality. This prevents new PIDs from accumulating over time.
+
+### Example
+
+ {
+ "CreateStream": {
+ "streamName": "big.query.data",
+ "objectName": "my-topic",
+ "spoutName": "KAFKA",
+ "spoutType": "TRANSACTIONAL",
+ "deserializer": "avro"
+ "options": {
+ "jw.sqe.spout.kafka.zkhosts": ["zk-01.host.com:2181","zk-02.host.com:2181"],
+ "jw.sqe.spout.kafka.clientid": "my-client-id",
+ "jw.sqe.spout.deserializer.avro.schemaname": "my.avro.schema"
+ }
+ }
+ }
+
+# Deserializers
+
+Deserializers allow us to take serialized or otherwise packed data that is emitted by a spout, parse it, and split its constituent parts into fields on the tuple. Kafka is a good example since it stores data packed as a message that can be formatted in any number of ways such as Avro. If no deserializer is specified, then the data remains on the stream as it is emitted by the spout without an pre-processing between the spout and the queries.
+
+## Avro
+
+The Avro deserializer takes an Avro record as a byte array along with a schema. Based on the fields needed by the queries, they are pulled from each record and added to each tuple. For example, if messages are stored in Kafka in Avro with the following schema:
+
+ {
+ "fields": [
+ {"name": "DateTime", "type": "string"},
+ {"name": "Embeds", "type": "int"},
+ {"name": "Plays", "type": "int"},
+ {"name": "Completes", "type": "int"},
+ {"name": "AdImpressions", "type": "int"},
+ {"name": "TimeWatched", "type": "long"}
+ ],
+ "name": "HourlyMeasures",
+ "namespace": "com.jwplayer.analytics.avro",
+ "type": "record"
+ }
+
+We then have a query that references the DateTime, Embeds, and Plays fields. The byte array message is then deserialized into an Avro record, then those 3 fields are added to the tuple as fields with the same name as the fields in the Avro schema.
+
+### Options
+
+* jw.sqe.spout.deserializer.avro.schemaname - The name of the avro schema to use to deserialize the byte array. The full namespace should be included. This should point to a Java object that implements SpecificRecord.
+**TODO: Allow specifying avro schemas either inline or from another file as another way of deserializing Avro objects**
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/7409c91f/docs/storm-sqe-using.md
----------------------------------------------------------------------
diff --git a/docs/storm-sqe-using.md b/docs/storm-sqe-using.md
new file mode 100644
index 0000000..e96ffc8
--- /dev/null
+++ b/docs/storm-sqe-using.md
@@ -0,0 +1,15 @@
+# Using SQE
+
+Using the SQE topology builder is straightforward. You just launch the topology using Storm with appropriate command line options:
+
+ storm jar sqe.jar com.jwplayer.sqe.Topology --config ./conf/conf.yaml --commands ./commands/commands1.json /my/commands/commands2.json --name=my-topology
+
+The locations of config and command files can be specified using a URI. For example
+
+ file:///conf/conf.yaml
+
+You can see all of the command line options by running:
+
+ java jar sqe.jar com.jwplayer.sqe.Topology --help
+
+Refer to the [Commands](storm-sqe-commands.html) section for more information on the JSON command format and the [Options](storm-sqe-options.html) section for more information on the options and config file format.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/7409c91f/docs/storm-sqe.md
----------------------------------------------------------------------
diff --git a/docs/storm-sqe.md b/docs/storm-sqe.md
new file mode 100644
index 0000000..f107761
--- /dev/null
+++ b/docs/storm-sqe.md
@@ -0,0 +1,28 @@
+# Streaming Query Engine
+
+SQE is a query engine written using Storm's Trident framework that takes a set of SQL-like commands, including queries, and runs them against one or more input streams. By using Trident, input streams are processed in micro-batches with both good latency and high throughput while guaranteeing "exactly-once" processing. Results can be returned through a list of output streams or SQE can handle persisting to a data store directly using one of its supported Trident states. SQE is designed to make it easy to query against large streams of data with good performance for many different use cases.
+
+* [Using SQE](storm-sqe-using.html)
+* [Options](storm-sqe-options.html)
+* [Commands](storm-sqe-commands.html)
+* [Expressions](storm-sqe-expressions.html)
+* [States](storm-sqe-states.html)
+* [Streams](storm-sqe-streams.html)
+* [Replay Filtering](storm-sqe-replay-filtering.html)
+
+## Potential Future Features
+
+* More expressions - String expressions, Average, etc.
+* Query planning and optimization
+* Better state/stream support and optimization
+* More options
+* Support for joins
+* Support for sub-queries, in-line queries and query chaining
+* Split out streams, states and other hard coded factories to configuration files. Factories should use the appropriate file to create appropriate objects. Then users can add additional expressions, states, etc. by building a jar with SQE as a dependency, then including their own versions of the configuration files with their new expressions, states, etc. (Note: we do this for expressions now)
+
+## Links
+
+* [Trident State](http://storm.apache.org/documentation/Trident-state.html) - Very useful for understanding how state and persisting data works in Trident
+* [Trident API Overview](http://storm.apache.org/documentation/Trident-API-Overview.html) - Basic overview of Trident. Not necessary to know to use SQE, but still helpful.
+* [Squall](https://github.com/epfldata/squall) - "A streaming / online query processing / analytics engine based on Apache Storm" - Another project from EPFL Data that does complex SQL like queries on top of Storm. Definitely something to keep an eye on, though use cases may be different. Doesn't support "exactly-once" processing yet, something we do by using Trident.
+
[04/11] storm git commit: initial SQE import
Posted by pt...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ExpandKeys.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ExpandKeys.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ExpandKeys.java
new file mode 100644
index 0000000..1e08781
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ExpandKeys.java
@@ -0,0 +1,19 @@
+package com.jwplayer.sqe.language.expression.transform;
+
+import org.apache.storm.trident.Stream;
+
+
+public class ExpandKeys extends TransformExpression {
+ public ExpandKeys() {
+ }
+
+ @Override
+ public Stream transform(Stream stream) {
+ return stream.each(getInputFields(), new com.jwplayer.sqe.trident.function.ExpandKeys(), getOutputFields());
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "expandkeys";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ExpandValues.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ExpandValues.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ExpandValues.java
new file mode 100644
index 0000000..9c3d331
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ExpandValues.java
@@ -0,0 +1,19 @@
+package com.jwplayer.sqe.language.expression.transform;
+
+import org.apache.storm.trident.Stream;
+
+
+public class ExpandValues extends TransformExpression {
+ public ExpandValues() {
+ }
+
+ @Override
+ public Stream transform(Stream stream) {
+ return stream.each(getInputFields(), new com.jwplayer.sqe.trident.function.ExpandValues(), getOutputFields());
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "expandvalues";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/FormatDate.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/FormatDate.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/FormatDate.java
new file mode 100644
index 0000000..dace3f5
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/FormatDate.java
@@ -0,0 +1,19 @@
+package com.jwplayer.sqe.language.expression.transform;
+
+import org.apache.storm.trident.Stream;
+
+
+public class FormatDate extends TransformExpression {
+ public FormatDate() {
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "formatdate";
+ }
+
+ @Override
+ public Stream transform(Stream stream) {
+ return stream.each(getInputFields(), new com.jwplayer.sqe.trident.function.FormatDate(), getOutputFields());
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/GetMapValue.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/GetMapValue.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/GetMapValue.java
new file mode 100644
index 0000000..2d50425
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/GetMapValue.java
@@ -0,0 +1,19 @@
+package com.jwplayer.sqe.language.expression.transform;
+
+import org.apache.storm.trident.Stream;
+
+
+public class GetMapValue extends TransformExpression{
+ public GetMapValue() {
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "getmapvalue";
+ }
+
+ @Override
+ public Stream transform(Stream stream) {
+ return stream.each(getInputFields(), new com.jwplayer.sqe.trident.function.GetMapValue(), getOutputFields());
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/GetTime.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/GetTime.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/GetTime.java
new file mode 100644
index 0000000..747a71a
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/GetTime.java
@@ -0,0 +1,19 @@
+package com.jwplayer.sqe.language.expression.transform;
+
+import org.apache.storm.trident.Stream;
+
+
+public class GetTime extends TransformExpression {
+ public GetTime() {
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "gettime";
+ }
+
+ @Override
+ public Stream transform(Stream stream) {
+ return stream.each(getInputFields(), new com.jwplayer.sqe.trident.function.GetTime(), getOutputFields());
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/Hash.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/Hash.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/Hash.java
new file mode 100644
index 0000000..f354a5f
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/Hash.java
@@ -0,0 +1,19 @@
+package com.jwplayer.sqe.language.expression.transform;
+
+import org.apache.storm.trident.Stream;
+
+
+public class Hash extends TransformExpression {
+ public Hash() {
+ }
+
+ @Override
+ public Stream transform(Stream stream) {
+ return stream.each(getInputFields(), new com.jwplayer.sqe.trident.function.Hash(), getOutputFields());
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "hash";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/If.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/If.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/If.java
new file mode 100644
index 0000000..fe61093
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/If.java
@@ -0,0 +1,20 @@
+package com.jwplayer.sqe.language.expression.transform;
+
+import com.jwplayer.sqe.trident.function.GetIf;
+import org.apache.storm.trident.Stream;
+
+
+public class If extends TransformExpression {
+ public If() {
+ }
+
+ @Override
+ public Stream transform(Stream stream) {
+ return stream.each(getInputFields(), new GetIf(), getOutputFields());
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "if";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ModulusArithmeticOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ModulusArithmeticOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ModulusArithmeticOperator.java
new file mode 100644
index 0000000..86b077f
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ModulusArithmeticOperator.java
@@ -0,0 +1,13 @@
+package com.jwplayer.sqe.language.expression.transform;
+
+
+public class ModulusArithmeticOperator extends ArithmeticOperator {
+ public ModulusArithmeticOperator() {
+ super(ArithmeticOperatorType.Modulus);
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "%";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/MultiplicationArithmeticOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/MultiplicationArithmeticOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/MultiplicationArithmeticOperator.java
new file mode 100644
index 0000000..232cedb
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/MultiplicationArithmeticOperator.java
@@ -0,0 +1,13 @@
+package com.jwplayer.sqe.language.expression.transform;
+
+
+public class MultiplicationArithmeticOperator extends ArithmeticOperator {
+ public MultiplicationArithmeticOperator() {
+ super(ArithmeticOperatorType.Multiplication);
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "*";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ParseDate.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ParseDate.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ParseDate.java
new file mode 100644
index 0000000..be74931
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ParseDate.java
@@ -0,0 +1,19 @@
+package com.jwplayer.sqe.language.expression.transform;
+
+import org.apache.storm.trident.Stream;
+
+
+public class ParseDate extends TransformExpression {
+ public ParseDate() {
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "parsedate";
+ }
+
+ @Override
+ public Stream transform(Stream stream) {
+ return stream.each(getInputFields(), new com.jwplayer.sqe.trident.function.ParseDate(), getOutputFields());
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/PlusArithmeticOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/PlusArithmeticOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/PlusArithmeticOperator.java
new file mode 100644
index 0000000..7268bea
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/PlusArithmeticOperator.java
@@ -0,0 +1,13 @@
+package com.jwplayer.sqe.language.expression.transform;
+
+
+public class PlusArithmeticOperator extends ArithmeticOperator {
+ public PlusArithmeticOperator() {
+ super(ArithmeticOperatorType.Addition);
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "+";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/RoundDate.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/RoundDate.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/RoundDate.java
new file mode 100644
index 0000000..5f4c02b
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/RoundDate.java
@@ -0,0 +1,19 @@
+package com.jwplayer.sqe.language.expression.transform;
+
+import org.apache.storm.trident.Stream;
+
+
+public class RoundDate extends TransformExpression {
+ public RoundDate() {
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "rounddate";
+ }
+
+ @Override
+ public Stream transform(Stream stream) {
+ return stream.each(getInputFields(), new com.jwplayer.sqe.trident.function.RoundDate(), getOutputFields());
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/SubtractionArithmeticOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/SubtractionArithmeticOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/SubtractionArithmeticOperator.java
new file mode 100644
index 0000000..c2d0c95
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/SubtractionArithmeticOperator.java
@@ -0,0 +1,13 @@
+package com.jwplayer.sqe.language.expression.transform;
+
+
+public class SubtractionArithmeticOperator extends ArithmeticOperator {
+ public SubtractionArithmeticOperator() {
+ super(ArithmeticOperatorType.Subtraction);
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "-";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/TransformExpression.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/TransformExpression.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/TransformExpression.java
new file mode 100644
index 0000000..2d4bb8c
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/TransformExpression.java
@@ -0,0 +1,18 @@
+package com.jwplayer.sqe.language.expression.transform;
+
+import com.jwplayer.sqe.language.expression.FunctionExpression;
+import com.jwplayer.sqe.language.expression.FunctionType;
+import org.apache.storm.trident.Stream;
+
+
+public abstract class TransformExpression extends FunctionExpression {
+ public TransformExpression() {
+ }
+
+ @Override
+ public FunctionType getFunctionType() {
+ return FunctionType.Transform;
+ }
+
+ public abstract Stream transform(Stream stream);
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/AndLogicalOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/AndLogicalOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/AndLogicalOperator.java
new file mode 100644
index 0000000..b8f82fa
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/AndLogicalOperator.java
@@ -0,0 +1,13 @@
+package com.jwplayer.sqe.language.expression.transform.predicate;
+
+
+public class AndLogicalOperator extends LogicalOperator {
+ public AndLogicalOperator() {
+ super(LogicalOperatorType.And);
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "and";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/CompareNumbers.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/CompareNumbers.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/CompareNumbers.java
new file mode 100644
index 0000000..d7e186b
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/CompareNumbers.java
@@ -0,0 +1,21 @@
+package com.jwplayer.sqe.language.expression.transform.predicate;
+
+import org.apache.storm.trident.Stream;
+
+
+public abstract class CompareNumbers extends PredicateExpression {
+ private NumberComparisonType comparisonType = null;
+
+ public CompareNumbers(NumberComparisonType comparisonType) {
+ super();
+ this.comparisonType = comparisonType;
+ }
+
+ @Override
+ public Stream transform(Stream stream) {
+ com.jwplayer.sqe.trident.function.CompareNumbers function =
+ new com.jwplayer.sqe.trident.function.CompareNumbers(comparisonType);
+
+ return stream.each(getInputFields(), function, getOutputFields());
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/Equals.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/Equals.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/Equals.java
new file mode 100644
index 0000000..7b6c123
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/Equals.java
@@ -0,0 +1,20 @@
+package com.jwplayer.sqe.language.expression.transform.predicate;
+
+import com.jwplayer.sqe.trident.function.CompareObjects;
+import org.apache.storm.trident.Stream;
+
+
+public class Equals extends PredicateExpression {
+ public Equals() {
+ }
+
+ @Override
+ public Stream transform(Stream stream) {
+ return stream.each(getInputFields(), new CompareObjects(), getOutputFields());
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "=";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/GreaterThanCompareNumbers.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/GreaterThanCompareNumbers.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/GreaterThanCompareNumbers.java
new file mode 100644
index 0000000..8431754
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/GreaterThanCompareNumbers.java
@@ -0,0 +1,13 @@
+package com.jwplayer.sqe.language.expression.transform.predicate;
+
+
+public class GreaterThanCompareNumbers extends CompareNumbers {
+ public GreaterThanCompareNumbers() {
+ super(NumberComparisonType.GreaterThan);
+ }
+
+ @Override
+ public String getFunctionName() {
+ return ">";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/GreaterThanEqualCompareNumbers.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/GreaterThanEqualCompareNumbers.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/GreaterThanEqualCompareNumbers.java
new file mode 100644
index 0000000..7663e44
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/GreaterThanEqualCompareNumbers.java
@@ -0,0 +1,13 @@
+package com.jwplayer.sqe.language.expression.transform.predicate;
+
+
+public class GreaterThanEqualCompareNumbers extends CompareNumbers {
+ public GreaterThanEqualCompareNumbers() {
+ super(NumberComparisonType.GreaterThanOrEqual);
+ }
+
+ @Override
+ public String getFunctionName() {
+ return ">=";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/In.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/In.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/In.java
new file mode 100644
index 0000000..4f9f771
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/In.java
@@ -0,0 +1,20 @@
+package com.jwplayer.sqe.language.expression.transform.predicate;
+
+import com.jwplayer.sqe.trident.function.IsIn;
+import org.apache.storm.trident.Stream;
+
+
+public class In extends PredicateExpression {
+ public In() {
+ }
+
+ @Override
+ public Stream transform(Stream stream) {
+ return stream.each(getInputFields(), new IsIn(), getOutputFields());
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "in";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LessThanCompareNumbers.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LessThanCompareNumbers.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LessThanCompareNumbers.java
new file mode 100644
index 0000000..d1e7223
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LessThanCompareNumbers.java
@@ -0,0 +1,13 @@
+package com.jwplayer.sqe.language.expression.transform.predicate;
+
+
+public class LessThanCompareNumbers extends CompareNumbers {
+ public LessThanCompareNumbers() {
+ super(NumberComparisonType.LessThan);
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "<";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LessThanEqualCompareNumbers.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LessThanEqualCompareNumbers.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LessThanEqualCompareNumbers.java
new file mode 100644
index 0000000..a8ce47c
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LessThanEqualCompareNumbers.java
@@ -0,0 +1,13 @@
+package com.jwplayer.sqe.language.expression.transform.predicate;
+
+
+public class LessThanEqualCompareNumbers extends CompareNumbers {
+ public LessThanEqualCompareNumbers() {
+ super(NumberComparisonType.LessThanOrEqual);
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "<=";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LogicalOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LogicalOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LogicalOperator.java
new file mode 100644
index 0000000..f1c64aa
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LogicalOperator.java
@@ -0,0 +1,19 @@
+package com.jwplayer.sqe.language.expression.transform.predicate;
+
+import com.jwplayer.sqe.trident.function.ProcessLogicalOperator;
+import org.apache.storm.trident.Stream;
+
+
+public abstract class LogicalOperator extends PredicateExpression {
+ LogicalOperatorType operatorType;
+
+ public LogicalOperator(LogicalOperatorType operatorType) {
+ super();
+ this.operatorType = operatorType;
+ }
+
+ @Override
+ public Stream transform(Stream stream) {
+ return stream.each(getInputFields(), new ProcessLogicalOperator(operatorType), getOutputFields());
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LogicalOperatorType.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LogicalOperatorType.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LogicalOperatorType.java
new file mode 100644
index 0000000..cf3e20c
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LogicalOperatorType.java
@@ -0,0 +1,8 @@
+package com.jwplayer.sqe.language.expression.transform.predicate;
+
+public enum LogicalOperatorType {
+ And,
+ Not,
+ Or,
+ Xor
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NotEqualCompareNumbers.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NotEqualCompareNumbers.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NotEqualCompareNumbers.java
new file mode 100644
index 0000000..f4ca8c6
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NotEqualCompareNumbers.java
@@ -0,0 +1,13 @@
+package com.jwplayer.sqe.language.expression.transform.predicate;
+
+
+public class NotEqualCompareNumbers extends CompareNumbers {
+ public NotEqualCompareNumbers() {
+ super(NumberComparisonType.NotEqual);
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "!=";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NotLogicalOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NotLogicalOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NotLogicalOperator.java
new file mode 100644
index 0000000..0925144
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NotLogicalOperator.java
@@ -0,0 +1,13 @@
+package com.jwplayer.sqe.language.expression.transform.predicate;
+
+
+public class NotLogicalOperator extends LogicalOperator {
+ public NotLogicalOperator() {
+ super(LogicalOperatorType.Not);
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "not";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NumberComparisonType.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NumberComparisonType.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NumberComparisonType.java
new file mode 100644
index 0000000..581cf40
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NumberComparisonType.java
@@ -0,0 +1,10 @@
+package com.jwplayer.sqe.language.expression.transform.predicate;
+
+public enum NumberComparisonType {
+ Equal,
+ GreaterThan,
+ GreaterThanOrEqual,
+ LessThan,
+ LessThanOrEqual,
+ NotEqual
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/OrLogicalOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/OrLogicalOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/OrLogicalOperator.java
new file mode 100644
index 0000000..80e24d3
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/OrLogicalOperator.java
@@ -0,0 +1,13 @@
+package com.jwplayer.sqe.language.expression.transform.predicate;
+
+
+public class OrLogicalOperator extends LogicalOperator {
+ public OrLogicalOperator() {
+ super(LogicalOperatorType.Or);
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "or";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/PredicateExpression.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/PredicateExpression.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/PredicateExpression.java
new file mode 100644
index 0000000..8523406
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/PredicateExpression.java
@@ -0,0 +1,9 @@
+package com.jwplayer.sqe.language.expression.transform.predicate;
+
+import com.jwplayer.sqe.language.expression.transform.TransformExpression;
+
+
+public abstract class PredicateExpression extends TransformExpression {
+ public PredicateExpression() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/RLike.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/RLike.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/RLike.java
new file mode 100644
index 0000000..a4debed
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/RLike.java
@@ -0,0 +1,20 @@
+package com.jwplayer.sqe.language.expression.transform.predicate;
+
+import com.jwplayer.sqe.trident.function.EvaluateRegularExpression;
+import org.apache.storm.trident.Stream;
+
+
+public class RLike extends PredicateExpression {
+ public RLike() {
+ }
+
+ @Override
+ public Stream transform(Stream stream) {
+ return stream.each(getInputFields(), new EvaluateRegularExpression(), getOutputFields());
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "rlike";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/XorLogicalOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/XorLogicalOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/XorLogicalOperator.java
new file mode 100644
index 0000000..ae722ce
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/XorLogicalOperator.java
@@ -0,0 +1,13 @@
+package com.jwplayer.sqe.language.expression.transform.predicate;
+
+
+public class XorLogicalOperator extends LogicalOperator {
+ public XorLogicalOperator() {
+ super(LogicalOperatorType.Xor);
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "xor";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/BaseDeserializer.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/BaseDeserializer.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/BaseDeserializer.java
new file mode 100644
index 0000000..720ce2d
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/BaseDeserializer.java
@@ -0,0 +1,25 @@
+package com.jwplayer.sqe.language.serde;
+
+import com.jwplayer.sqe.language.serde.avro.AvroDeserializer;
+import com.jwplayer.sqe.language.serde.avro.AvroDeserializerOptions;
+import org.apache.storm.trident.Stream;
+import org.apache.storm.tuple.Fields;
+
+import java.util.Map;
+
+
+public abstract class BaseDeserializer {
+ public abstract Stream deserialize(Stream stream, Fields requiredFields);
+
+ public static BaseDeserializer makeDeserializer(String deserializerName, Map deserializerOptions) {
+ switch(deserializerName.toLowerCase()) {
+ case "avro":
+ AvroDeserializerOptions avroOptions = AvroDeserializerOptions.parse(deserializerOptions);
+ return new AvroDeserializer(avroOptions);
+ case "identity":
+ return new IdentityDeserializer();
+ default:
+ throw new RuntimeException(deserializerName + " is not a supported deserializer");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/IdentityDeserializer.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/IdentityDeserializer.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/IdentityDeserializer.java
new file mode 100644
index 0000000..76fc262
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/IdentityDeserializer.java
@@ -0,0 +1,16 @@
+package com.jwplayer.sqe.language.serde;
+
+
+import org.apache.storm.trident.Stream;
+import org.apache.storm.tuple.Fields;
+
+public class IdentityDeserializer extends BaseDeserializer {
+ public void IdentityDeserializer() {
+
+ }
+
+ @Override
+ public Stream deserialize(Stream stream, Fields requiredFields) {
+ return stream;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/avro/AvroDeserializer.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/avro/AvroDeserializer.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/avro/AvroDeserializer.java
new file mode 100644
index 0000000..f5868f4
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/avro/AvroDeserializer.java
@@ -0,0 +1,21 @@
+package com.jwplayer.sqe.language.serde.avro;
+
+import com.jwplayer.sqe.language.serde.BaseDeserializer;
+import com.jwplayer.sqe.trident.function.GetTupleFromAvro;
+import org.apache.storm.trident.Stream;
+import org.apache.storm.tuple.Fields;
+
+
+public class AvroDeserializer extends BaseDeserializer {
+ AvroDeserializerOptions options;
+
+ public AvroDeserializer(AvroDeserializerOptions options) {
+ this.options = options;
+ }
+
+ @Override
+ public Stream deserialize(Stream stream, Fields requiredFields) {
+ return stream
+ .each(stream.getOutputFields(), new GetTupleFromAvro(options.schemaName, requiredFields), requiredFields);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/avro/AvroDeserializerOptions.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/avro/AvroDeserializerOptions.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/avro/AvroDeserializerOptions.java
new file mode 100644
index 0000000..3abd155
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/avro/AvroDeserializerOptions.java
@@ -0,0 +1,16 @@
+package com.jwplayer.sqe.language.serde.avro;
+
+import java.util.Map;
+
+public class AvroDeserializerOptions {
+ public String schemaName = null;
+
+ public static AvroDeserializerOptions parse(Map map) {
+ AvroDeserializerOptions options = new AvroDeserializerOptions();
+
+ if(map.containsKey("jw.sqe.spout.deserializer.avro.schemaname"))
+ options.schemaName = (String) map.get("jw.sqe.spout.deserializer.avro.schemaname");
+
+ return options;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/StateAdapter.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/StateAdapter.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/StateAdapter.java
new file mode 100644
index 0000000..0483a5b
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/StateAdapter.java
@@ -0,0 +1,41 @@
+package com.jwplayer.sqe.language.state;
+
+import com.jwplayer.sqe.language.state.kafka.KafkaStateAdapter;
+import com.jwplayer.sqe.language.state.kafka.KafkaStateOptions;
+import com.jwplayer.sqe.language.state.mongodb.MongoDBStateAdapter;
+import com.jwplayer.sqe.language.state.redis.RedisStateAdapter;
+import com.jwplayer.sqe.language.state.redis.RedisStateOptions;
+import com.jwplayer.sqe.trident.state.mongodb.MongoDBStateOptions;
+import org.apache.storm.trident.Stream;
+import org.apache.storm.trident.TridentState;
+import org.apache.storm.trident.state.StateFactory;
+import org.apache.storm.trident.state.StateType;
+import org.apache.storm.tuple.Fields;
+
+import java.util.List;
+import java.util.Map;
+
+
+public abstract class StateAdapter {
+ public abstract StateFactory makeFactory(String objectName, List<String> keyFields, String valueField, StateType stateType, StateOperationType stateOperationType);
+
+ public TridentState partitionPersist(Stream stream, StateFactory stateFactory, Fields keyFields) {
+ throw new UnsupportedOperationException("This adapter does not support partition persist");
+ }
+
+ public static StateAdapter makeAdapter(String stateName, Map options) {
+ switch (stateName) {
+ case "mongo":
+ MongoDBStateOptions mongoDBStateOptions = MongoDBStateOptions.parse(options);
+ return new MongoDBStateAdapter(mongoDBStateOptions);
+ case "redis":
+ RedisStateOptions redisStateOptions = RedisStateOptions.parse(options);
+ return new RedisStateAdapter(redisStateOptions);
+ case "kafka":
+ KafkaStateOptions kafkaStateOptions = KafkaStateOptions.parse(options);
+ return new KafkaStateAdapter(kafkaStateOptions);
+ default:
+ throw new RuntimeException("Unknown state name: " + stateName);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/StateOperationType.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/StateOperationType.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/StateOperationType.java
new file mode 100644
index 0000000..4b56fd2
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/StateOperationType.java
@@ -0,0 +1,6 @@
+package com.jwplayer.sqe.language.state;
+
+public enum StateOperationType {
+ AGGREGATE,
+ NONAGGREGATE
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/JwTridentKafkaStateFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/JwTridentKafkaStateFactory.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/JwTridentKafkaStateFactory.java
new file mode 100644
index 0000000..9cc927e
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/JwTridentKafkaStateFactory.java
@@ -0,0 +1,75 @@
+package com.jwplayer.sqe.language.state.kafka;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.storm.kafka.trident.TridentKafkaState;
+import org.apache.storm.kafka.trident.mapper.TridentTupleToKafkaMapper;
+import org.apache.storm.kafka.trident.selector.KafkaTopicSelector;
+import org.apache.storm.trident.state.State;
+import org.apache.storm.trident.state.StateFactory;
+import org.apache.storm.task.IMetricsContext;
+
+
+public class JwTridentKafkaStateFactory implements StateFactory {
+
+ private static final Logger LOG = LoggerFactory.getLogger(JwTridentKafkaStateFactory.class);
+
+ private TridentTupleToKafkaMapper mapper;
+ private KafkaTopicSelector topicSelector;
+ public KafkaStateOptions kafka_options;
+
+ public JwTridentKafkaStateFactory withTridentTupleToKafkaMapper(TridentTupleToKafkaMapper mapper) {
+ this.mapper = mapper;
+ return this;
+ }
+
+ public JwTridentKafkaStateFactory withKafkaOptions(KafkaStateOptions options) {
+ kafka_options = options;
+ return this;
+ }
+
+ public JwTridentKafkaStateFactory withKafkaTopicSelector(KafkaTopicSelector selector) {
+ this.topicSelector = selector;
+ return this;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
+ LOG.info("makeState(partitonIndex={}, numpartitions={}", partitionIndex, numPartitions);
+
+ TridentKafkaState state = new TridentKafkaState()
+ .withKafkaTopicSelector(this.topicSelector)
+ .withTridentTupleToKafkaMapper(this.mapper);
+ Properties kafkaConf = new Properties();
+ for(Map.Entry entry: (Set<Map.Entry>) this.getconf(conf).entrySet()) {
+ if(entry.getValue() != null) {
+ kafkaConf.put(entry.getKey(), entry.getValue());
+ }
+ }
+ state.prepare(kafkaConf);
+ return state;
+ }
+
+ @SuppressWarnings("unchecked")
+ public Map getconf(Map conf) {
+ KafkaStateOptions sqeKafkaOptions = kafka_options;
+ Map kafkaConf = new HashMap();
+ kafkaConf.putAll(conf);
+ // Storm 1.0 uses the Kafka 0.9 producer, which has different names for the configs
+ kafkaConf.put("bootstrap.servers", sqeKafkaOptions.metadataBrokerList);
+ kafkaConf.put("acks", sqeKafkaOptions.requestRequiredAcks);
+ kafkaConf.put("producer.type", sqeKafkaOptions.producerType);
+ kafkaConf.put("key.serializer",sqeKafkaOptions.KeyserializerClass);
+ kafkaConf.put("value.serializer", sqeKafkaOptions.serializerClass);
+ kafkaConf.put("partitioner.class", sqeKafkaOptions.partitionClass);
+
+ return kafkaConf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/KafkaStateAdapter.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/KafkaStateAdapter.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/KafkaStateAdapter.java
new file mode 100644
index 0000000..71ac874
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/KafkaStateAdapter.java
@@ -0,0 +1,76 @@
+package com.jwplayer.sqe.language.state.kafka;
+
+import java.util.ArrayList;
+import java.util.List;
+
+
+import com.jwplayer.sqe.language.state.StateAdapter;
+import com.jwplayer.sqe.language.state.StateOperationType;
+import com.jwplayer.sqe.language.stream.StreamAdapter;
+import com.jwplayer.sqe.trident.function.ConvertMetadataToBytes;
+import com.jwplayer.sqe.trident.function.Hash;
+import org.apache.storm.kafka.trident.TridentKafkaUpdater;
+import org.apache.storm.kafka.trident.mapper.FieldNameBasedTupleToKafkaMapper;
+import org.apache.storm.kafka.trident.selector.DefaultTopicSelector;
+import org.apache.storm.trident.Stream;
+import org.apache.storm.trident.TridentState;
+import org.apache.storm.trident.state.StateFactory;
+import org.apache.storm.trident.state.StateType;
+import org.apache.storm.tuple.Fields;
+
+
+public class KafkaStateAdapter extends StateAdapter {
+ public static final String KAFKA_KEY = "_kafka_state_key";
+ protected KafkaStateOptions options ;
+
+ public KafkaStateAdapter(KafkaStateOptions options) {
+ this.options = options;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public StateFactory makeFactory(String objectName, List<String> keyFields, String valueField, StateType stateType, StateOperationType stateOperationType) {
+ return getTridentStateFactory(objectName);
+ }
+
+ protected JwTridentKafkaStateFactory getTridentStateFactory(String topic) {
+ JwTridentKafkaStateFactory factory = new JwTridentKafkaStateFactory();
+ factory.withKafkaOptions(options).withKafkaTopicSelector(new DefaultTopicSelector(topic));
+
+ return factory;
+ }
+
+ public TridentState partitionPersist(Stream stream, StateFactory stateFactory, Fields keyFields) {
+ JwTridentKafkaStateFactory factory = (JwTridentKafkaStateFactory) stateFactory;
+ List<String> persistFields = new ArrayList<>();
+ for(String fieldName: keyFields) persistFields.add(fieldName);
+
+ if(options.KeyType.equals("field")) {
+ factory = factory.withTridentTupleToKafkaMapper(
+ new FieldNameBasedTupleToKafkaMapper<byte[], byte[]>(keyFields.get(0), keyFields.get(1)));
+ } else {
+ factory = factory.withTridentTupleToKafkaMapper(
+ new FieldNameBasedTupleToKafkaMapper<byte[], byte[]>(KafkaStateAdapter.KAFKA_KEY, keyFields.get(0)));
+ persistFields.add(KafkaStateAdapter.KAFKA_KEY);
+ }
+
+ switch (options.KeyType) {
+ case "field":
+ break;
+ case "messagehash":
+ stream = stream.each(new Fields(keyFields.get(0)), new Hash(), new Fields(KafkaStateAdapter.KAFKA_KEY));
+ break;
+ case "streammetadata":
+ stream = stream.each(
+ new Fields(StreamAdapter.STREAM_METADATA_FIELD),
+ new ConvertMetadataToBytes(),
+ new Fields(KafkaStateAdapter.KAFKA_KEY)
+ );
+ break;
+ default:
+ throw new IllegalArgumentException(options.KeyType + " is not a supported key type");
+ }
+
+ return stream.partitionPersist(factory, new Fields(persistFields), new TridentKafkaUpdater());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/KafkaStateOptions.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/KafkaStateOptions.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/KafkaStateOptions.java
new file mode 100644
index 0000000..9093c11
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/KafkaStateOptions.java
@@ -0,0 +1,51 @@
+package com.jwplayer.sqe.language.state.kafka;
+
+import com.google.common.base.Joiner;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+public class KafkaStateOptions implements Serializable {
+ public String serializerClass = "org.apache.kafka.common.serialization.ByteArraySerializer";
+ public String metadataBrokerList;
+ public String producerType = "async";
+ public String requestRequiredAcks = "1";
+ public String partitionClass = "org.apache.kafka.clients.producer.internals.DefaultPartitioner";
+ public String KeyserializerClass = "org.apache.kafka.common.serialization.ByteArraySerializer";
+ public String KeyType = "messagehash";
+
+ @SuppressWarnings("unchecked")
+ public static KafkaStateOptions parse(Map map) {
+ KafkaStateOptions options = new KafkaStateOptions();
+
+ if (map.containsKey("jw.sqe.state.kafka.brokers"))
+ options.metadataBrokerList =
+ Joiner.on(',').join((List<String>) map.get("jw.sqe.state.kafka.brokers"));
+
+ if (map.containsKey("jw.sqe.state.kafka.serializerClass"))
+ options.serializerClass =
+ (String) map.get("jw.sqe.state.kafka.serializerClass");
+
+ if (map.containsKey("jw.sqe.state.kafka.key.serializerClass"))
+ options.KeyserializerClass =
+ (String) map.get("jw.sqe.state.kafka.key.serializerClass");
+
+ if (map.containsKey("jw.sqe.state.kafka.partitionClass"))
+ options.partitionClass =
+ (String) map.get("jw.sqe.state.kafka.partitionClass");
+
+ if (map.containsKey("jw.sqe.state.kafka.producerType"))
+ options.producerType =
+ (String) map.get("jw.sqe.state.kafka.producerType");
+
+ if (map.containsKey("jw.sqe.state.kafka.request.requiredAcks"))
+ options.requestRequiredAcks =
+ (String) map.get("jw.sqe.state.kafka.request.requiredAcks");
+
+ if (map.containsKey("jw.sqe.state.kafka.keytype"))
+ options.KeyType = ((String) map.get("jw.sqe.state.kafka.keytype")).toLowerCase();
+
+ return options;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/SourcePartitionPartitioner.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/SourcePartitionPartitioner.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/SourcePartitionPartitioner.java
new file mode 100644
index 0000000..38074b2
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/SourcePartitionPartitioner.java
@@ -0,0 +1,34 @@
+package com.jwplayer.sqe.language.state.kafka;
+
+
+import com.jwplayer.sqe.trident.StreamMetadata;
+
+
+import org.apache.kafka.clients.producer.Partitioner;
+import org.apache.kafka.common.Cluster;
+import org.apache.kafka.common.PartitionInfo;
+
+import java.util.List;
+import java.util.Map;
+
+public class SourcePartitionPartitioner implements Partitioner {
+ @Override
+ public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {
+ List<PartitionInfo> partitions = cluster.partitionsForTopic(topic);
+ int numPartitions = partitions.size();
+ StreamMetadata metadata = StreamMetadata.parseBytes(keyBytes);
+ int sourcePartition = metadata.partition;
+
+ return sourcePartition % numPartitions;
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ @Override
+ public void configure(Map<String, ?> map) {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/mongodb/MongoDBStateAdapter.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/mongodb/MongoDBStateAdapter.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/mongodb/MongoDBStateAdapter.java
new file mode 100644
index 0000000..6ef1c88
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/mongodb/MongoDBStateAdapter.java
@@ -0,0 +1,31 @@
+package com.jwplayer.sqe.language.state.mongodb;
+
+import com.jwplayer.sqe.language.state.StateAdapter;
+import com.jwplayer.sqe.language.state.StateOperationType;
+import com.jwplayer.sqe.trident.state.mongodb.MongoDBState;
+import com.jwplayer.sqe.trident.state.mongodb.MongoDBStateOptions;
+
+import org.apache.storm.trident.state.StateFactory;
+import org.apache.storm.trident.state.StateType;
+
+import java.util.List;
+
+public class MongoDBStateAdapter extends StateAdapter {
+ private MongoDBStateOptions options;
+
+ public MongoDBStateAdapter(MongoDBStateOptions options) { this.options = options; }
+
+ @Override
+ public StateFactory makeFactory(String objectName, List<String> keyFields, String valueField, StateType stateType, StateOperationType stateOperationType) {
+ switch(stateType) {
+ case NON_TRANSACTIONAL:
+ return MongoDBState.nonTransactional(objectName, keyFields, valueField, options);
+ case OPAQUE:
+ return MongoDBState.opaque(objectName, keyFields, valueField, options);
+ case TRANSACTIONAL:
+ return MongoDBState.transactional(objectName, keyFields, valueField, options);
+ default:
+ throw new RuntimeException("Unknown state type: " + stateType);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateAdapter.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateAdapter.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateAdapter.java
new file mode 100644
index 0000000..986878b
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateAdapter.java
@@ -0,0 +1,142 @@
+package com.jwplayer.sqe.language.state.redis;
+
+import com.jwplayer.sqe.trident.state.GsonOpaqueSerializer;
+import com.jwplayer.sqe.trident.state.GsonTransactionalSerializer;
+import com.jwplayer.sqe.language.state.StateAdapter;
+import com.jwplayer.sqe.language.state.StateOperationType;
+import com.jwplayer.sqe.trident.state.redis.RedisKeyFactory;
+
+import org.apache.storm.redis.common.config.JedisPoolConfig;
+import org.apache.storm.redis.common.mapper.RedisStoreMapper;
+import org.apache.storm.redis.trident.state.Options;
+import org.apache.storm.redis.trident.state.RedisDataTypeDescription;
+import org.apache.storm.redis.trident.state.RedisMapState;
+import org.apache.storm.redis.trident.state.RedisStateUpdater;
+import org.apache.storm.trident.Stream;
+import org.apache.storm.trident.TridentState;
+import org.apache.storm.trident.state.StateFactory;
+import org.apache.storm.trident.state.StateType;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.ITuple;
+
+import java.util.ArrayList;
+import java.util.List;
+
+
+public class RedisStateAdapter extends StateAdapter {
+ protected RedisStateOptions options;
+
+ public RedisStateAdapter(RedisStateOptions options) { this.options = options; }
+
+ @Override
+ public TridentState partitionPersist(Stream stream, StateFactory stateFactory, Fields keyFields) {
+ RedisStateFactory factory = (RedisStateFactory) stateFactory;
+ RedisStateUpdater updater = new RedisStateUpdater(new StoreMapper(this.options,
+ factory.objectName, factory.streamFields));
+ updater.setExpireInterval(options.expireintervalsec);
+ return stream.partitionPersist(stateFactory, keyFields, updater);
+ }
+
+ private static class StoreMapper implements RedisStoreMapper {
+ private static final long serialVersionUID = 1L;
+ private RedisKeyFactory keyStringFactory;
+ private RedisKeyFactory valueStringFactory;
+ private org.apache.storm.redis.common.mapper.RedisDataTypeDescription description;
+
+ public StoreMapper(RedisStateOptions options, String objectName, List<String> streamFields) {
+ // No additional key since not aggregating
+ description = new org.apache.storm.redis.common.mapper.RedisDataTypeDescription(
+ org.apache.storm.redis.common.mapper.RedisDataTypeDescription.RedisDataType.SET,
+ objectName);
+
+ List<Integer> keyNameFieldIndexes = new ArrayList<>();
+ List<Integer> fieldNameFieldIndexes = new ArrayList<>();
+
+ for(int i = 0; i < streamFields.size(); i++) {
+ if(options.keyNameFields.contains(streamFields.get(i))) keyNameFieldIndexes.add(i);
+ if(options.fieldNameFields.contains(streamFields.get(i))) fieldNameFieldIndexes.add(i);
+ }
+ keyStringFactory = new RedisKeyFactory(options.delimiter, keyNameFieldIndexes, "", "");
+ valueStringFactory = new RedisKeyFactory(options.delimiter, fieldNameFieldIndexes, "", "");
+ }
+
+ @Override
+ public org.apache.storm.redis.common.mapper.RedisDataTypeDescription getDataTypeDescription() {
+ return description;
+ }
+
+ @Override
+ public String getKeyFromTuple(ITuple tuple) {
+ return keyStringFactory.build(tuple.getValues());
+ }
+
+ @Override
+ public String getValueFromTuple(ITuple tuple) {
+ return valueStringFactory.build(tuple.getValues());
+ }
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public StateFactory makeFactory(String objectName, List<String> keyFields, String valueField, StateType stateType, StateOperationType stateOperationType) {
+ JedisPoolConfig poolConfig =
+ new JedisPoolConfig.Builder()
+ .setDatabase(options.database)
+ .setHost(options.host)
+ .setPort(options.port)
+ .setTimeout(15000)
+ .build();
+ Options rOptions = new Options();
+
+ // Build the Redis options based on configuration from SQE
+ switch(options.dataType) {
+ case STRING:
+ List<Integer> indexes = new ArrayList<>();
+
+ for(int i = 0; i < keyFields.size(); i++) indexes.add(i);
+
+ rOptions.keyFactory = new RedisKeyFactory(options.delimiter, indexes, objectName, valueField);
+ rOptions.dataTypeDescription = new RedisDataTypeDescription(options.dataType);
+ break;
+ case HASH:
+ case SET:
+ List<Integer> keyNameFieldIndexes = new ArrayList<>();
+ List<Integer> fieldNameFieldIndexes = new ArrayList<>();
+
+ for(int i = 0; i < keyFields.size(); i++) {
+ if(options.keyNameFields.contains(keyFields.get(i))) keyNameFieldIndexes.add(i);
+ if(options.fieldNameFields.contains(keyFields.get(i))) fieldNameFieldIndexes.add(i);
+ }
+
+ rOptions.keyFactory = new RedisKeyFactory(options.delimiter, keyNameFieldIndexes, objectName, "");
+ rOptions.dataTypeDescription =
+ new RedisDataTypeDescription(options.dataType,
+ new RedisKeyFactory(options.delimiter, fieldNameFieldIndexes, "", valueField));
+ break;
+ case LIST:
+ case SORTED_SET:
+ case HYPER_LOG_LOG:
+ throw new IllegalArgumentException("Unsupported Redis data type: " + options.dataType);
+ }
+
+ // Set TTL for the keys
+ rOptions.expireIntervalSec = options.expireintervalsec;
+
+ if (stateOperationType == StateOperationType.NONAGGREGATE){
+ return new RedisStateFactory(poolConfig, objectName, keyFields);
+ }
+ // Return the appropriate state factory
+ switch(stateType) {
+ case NON_TRANSACTIONAL:
+ return RedisMapState.nonTransactional(poolConfig, rOptions);
+ case OPAQUE:
+ rOptions.serializer = new GsonOpaqueSerializer();
+ return RedisMapState.opaque(poolConfig, rOptions);
+ case TRANSACTIONAL:
+ rOptions.serializer = new GsonTransactionalSerializer();
+ return RedisMapState.transactional(poolConfig, rOptions);
+ default:
+ throw new RuntimeException("Unknown state type: " + stateType);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateFactory.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateFactory.java
new file mode 100644
index 0000000..9faa980
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateFactory.java
@@ -0,0 +1,29 @@
+package com.jwplayer.sqe.language.state.redis;
+
+import java.util.List;
+
+import org.apache.storm.redis.common.config.JedisPoolConfig;
+import org.apache.storm.redis.trident.state.RedisState;
+
+/**
+ * Similar to RedisState.Factory but also allows storing an objectName,
+ * used to name-space hash data in Redis
+ */
+public class RedisStateFactory extends RedisState.Factory{
+ private static final long serialVersionUID = 1L;
+ public List<String> streamFields;
+ public String objectName;
+
+
+ /**
+ * Store objectName for persisting state to Hash (additional key)
+ * @param config
+ * @param objectName
+ */
+ public RedisStateFactory(JedisPoolConfig config, String objectName, List<String> streamFields) {
+ super(config);
+ this.objectName = objectName;
+ this.streamFields = streamFields;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateOptions.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateOptions.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateOptions.java
new file mode 100644
index 0000000..3699480
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateOptions.java
@@ -0,0 +1,43 @@
+package com.jwplayer.sqe.language.state.redis;
+
+import org.apache.storm.redis.trident.state.RedisDataTypeDescription.RedisDataType;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class RedisStateOptions implements Serializable {
+ public int database = 0;
+ public RedisDataType dataType = RedisDataType.STRING;
+ public String delimiter = ":";
+ public List<String> fieldNameFields = new ArrayList<>();
+ public String host = "";
+ public List<String> keyNameFields = new ArrayList<>();
+ public int port = 6379;
+ public int expireintervalsec = 0;
+
+ @SuppressWarnings("unchecked")
+ public static RedisStateOptions parse(Map map) {
+ RedisStateOptions options = new RedisStateOptions();
+
+ if(map.containsKey("jw.sqe.state.redis.database"))
+ options.database = ((Number) map.get("jw.sqe.state.redis.database")).intValue();
+ if(map.containsKey("jw.sqe.state.redis.datatype"))
+ options.dataType = RedisDataType.valueOf((String) map.get("jw.sqe.state.redis.datatype"));
+ if(map.containsKey("jw.sqe.state.redis.delimiter"))
+ options.delimiter = (String) map.get("jw.sqe.state.redis.delimiter");
+ if(map.containsKey("jw.sqe.state.redis.expireintervalsec"))
+ options.expireintervalsec = ((Number)map.get("jw.sqe.state.redis.expireintervalsec")).intValue();
+ if(map.containsKey("jw.sqe.state.redis.fieldname.fields"))
+ options.fieldNameFields = (List<String>) map.get("jw.sqe.state.redis.fieldname.fields");
+ if(map.containsKey("jw.sqe.state.redis.host"))
+ options.host = (String) map.get("jw.sqe.state.redis.host");
+ if(map.containsKey("jw.sqe.state.redis.keyname.fields"))
+ options.keyNameFields = (List<String>) map.get("jw.sqe.state.redis.keyname.fields");
+ if(map.containsKey("jw.sqe.state.redis.port"))
+ options.port = Integer.parseInt((String) map.get("jw.sqe.state.redis.port"));
+
+ return options;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/InputStreamType.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/InputStreamType.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/InputStreamType.java
new file mode 100644
index 0000000..7bebbd6
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/InputStreamType.java
@@ -0,0 +1,6 @@
+package com.jwplayer.sqe.language.stream;
+
+public enum InputStreamType {
+ BinaryAvro,
+ Tuple
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/StreamAdapter.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/StreamAdapter.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/StreamAdapter.java
new file mode 100644
index 0000000..f43c35a
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/StreamAdapter.java
@@ -0,0 +1,36 @@
+package com.jwplayer.sqe.language.stream;
+
+import com.clearspring.analytics.hash.MurmurHash;
+import com.jwplayer.sqe.language.stream.kafka.KafkaStreamAdapter;
+import com.jwplayer.sqe.language.stream.kafka.KafkaStreamAdapterOptions;
+import com.jwplayer.sqe.language.stream.testing.FixedBatchSpoutOptions;
+import com.jwplayer.sqe.language.stream.testing.FixedBatchSpoutStreamAdapter;
+import org.apache.storm.trident.Stream;
+import org.apache.storm.trident.TridentTopology;
+import org.apache.storm.trident.state.StateType;
+
+import java.util.Map;
+
+
+public abstract class StreamAdapter {
+ public abstract Stream makeStream(TridentTopology topology, String topologyName, String streamName, String objectName, StateType spoutType);
+
+ public static final String STREAM_METADATA_FIELD = "_stream_metadata";
+
+ public static Long createPid(String topologyName, String streamName, String secondaryID) {
+ return MurmurHash.hash64(topologyName + streamName + secondaryID);
+ }
+
+ public static StreamAdapter makeAdapter(String spoutName, Map options) {
+ switch(spoutName.toLowerCase()) {
+ case "fixed":
+ FixedBatchSpoutOptions fixedOptions = FixedBatchSpoutOptions.parse(options);
+ return new FixedBatchSpoutStreamAdapter(fixedOptions);
+ case "kafka":
+ KafkaStreamAdapterOptions kafkaOptions = KafkaStreamAdapterOptions.parse(options);
+ return new KafkaStreamAdapter(kafkaOptions);
+ default:
+ throw new RuntimeException("Unknown spout name: " + spoutName);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/StreamContainer.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/StreamContainer.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/StreamContainer.java
new file mode 100644
index 0000000..8f15ac1
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/StreamContainer.java
@@ -0,0 +1,15 @@
+package com.jwplayer.sqe.language.stream;
+
+import org.apache.storm.trident.Stream;
+
+public class StreamContainer {
+ public InputStreamType inputStreamType;
+ public Stream stream;
+ public String streamName;
+
+ public StreamContainer(InputStreamType inputStreamType, Stream stream, String streamName) {
+ this.inputStreamType = inputStreamType;
+ this.stream = stream;
+ this.streamName = streamName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/kafka/KafkaStreamAdapter.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/kafka/KafkaStreamAdapter.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/kafka/KafkaStreamAdapter.java
new file mode 100644
index 0000000..e3e5c91
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/kafka/KafkaStreamAdapter.java
@@ -0,0 +1,52 @@
+package com.jwplayer.sqe.language.stream.kafka;
+
+import com.jwplayer.sqe.language.stream.StreamAdapter;
+import com.jwplayer.sqe.trident.spout.kafka.FilteredOpaqueTridentKafkaSpout;
+import com.jwplayer.sqe.trident.spout.kafka.FilteredTransactionalTridentKafkaSpout;
+import org.apache.storm.kafka.trident.OpaqueTridentKafkaSpout;
+import org.apache.storm.kafka.trident.TransactionalTridentKafkaSpout;
+import org.apache.storm.trident.Stream;
+import org.apache.storm.trident.TridentTopology;
+import org.apache.storm.trident.state.StateType;
+
+
+public class KafkaStreamAdapter extends StreamAdapter {
+ KafkaStreamAdapterOptions options;
+
+ public KafkaStreamAdapter(KafkaStreamAdapterOptions options) {
+ this.options = options;
+ }
+
+ @Override
+ public Stream makeStream(TridentTopology topology, String topologyName, String streamName, String objectName, StateType spoutType) {
+ String txID = topologyName + "/" + streamName;
+
+ if (options.filterReplays) {
+ switch (spoutType) {
+ case TRANSACTIONAL:
+ FilteredTransactionalTridentKafkaSpout tSpout =
+ new FilteredTransactionalTridentKafkaSpout(options.getKafkaConfig(topologyName, streamName, objectName), options.filterReplaysMetadataTtl);
+ return topology.newStream(txID, tSpout);
+ case OPAQUE:
+ FilteredOpaqueTridentKafkaSpout oSpout =
+ new FilteredOpaqueTridentKafkaSpout(options.getKafkaConfig(topologyName, streamName, objectName), options.filterReplaysMetadataTtl);
+ return topology.newStream(txID, oSpout);
+ default:
+ throw new RuntimeException(spoutType.toString() + " is not a supported state type");
+ }
+ } else {
+ switch (spoutType) {
+ case TRANSACTIONAL:
+ TransactionalTridentKafkaSpout tSpout =
+ new TransactionalTridentKafkaSpout(options.getKafkaConfig(topologyName, streamName, objectName));
+ return topology.newStream(txID, tSpout);
+ case OPAQUE:
+ OpaqueTridentKafkaSpout oSpout =
+ new OpaqueTridentKafkaSpout(options.getKafkaConfig(topologyName, streamName, objectName));
+ return topology.newStream(txID, oSpout);
+ default:
+ throw new RuntimeException(spoutType.toString() + " is not a supported state type");
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/kafka/KafkaStreamAdapterOptions.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/kafka/KafkaStreamAdapterOptions.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/kafka/KafkaStreamAdapterOptions.java
new file mode 100644
index 0000000..2c70053
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/kafka/KafkaStreamAdapterOptions.java
@@ -0,0 +1,54 @@
+package com.jwplayer.sqe.language.stream.kafka;
+
+import com.google.common.base.Joiner;
+import com.jwplayer.sqe.trident.spout.kafka.SqeRawFullScheme;
+import org.apache.storm.kafka.FullSchemeAsMultiScheme;
+import org.apache.storm.kafka.ZkHosts;
+import org.apache.storm.kafka.trident.TridentKafkaConfig;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+
+public class KafkaStreamAdapterOptions implements Serializable {
+ Integer bufferSizeBytes = null;
+ String clientID;
+ Integer fetchSizeBytes = null;
+ Boolean filterReplays = false;
+ Long filterReplaysMetadataTtl = 60L * 60L * 24L * 2L;
+ Long maxOffsetBehind = null;
+ ZkHosts zkHosts;
+
+ public TridentKafkaConfig getKafkaConfig(String topologyName, String streamName, String topic) {
+ TridentKafkaConfig config = new TridentKafkaConfig(zkHosts, topic, clientID);
+ if(bufferSizeBytes != null) config.bufferSizeBytes = bufferSizeBytes;
+ if(fetchSizeBytes != null) config.fetchSizeBytes = fetchSizeBytes;
+ if(maxOffsetBehind != null) config.maxOffsetBehind = maxOffsetBehind;
+ config.scheme = new FullSchemeAsMultiScheme(
+ new SqeRawFullScheme(topologyName, streamName, zkHosts)
+ );
+
+ return config;
+ }
+
+ @SuppressWarnings("unchecked")
+ public static KafkaStreamAdapterOptions parse(Map map) {
+ KafkaStreamAdapterOptions options = new KafkaStreamAdapterOptions();
+ options.zkHosts = new ZkHosts(Joiner.on(',').join((List<String>) map.get("jw.sqe.spout.kafka.zkhosts")));
+ options.clientID = (String) map.get("jw.sqe.spout.kafka.clientid");
+
+ if(map.containsKey("jw.sqe.spout.kafka.bufferSizeBytes"))
+ options.bufferSizeBytes = (int) map.get("jw.sqe.spout.kafka.bufferSizeBytes");
+ if(map.containsKey("jw.sqe.spout.kafka.fetchSizeBytes"))
+ options.fetchSizeBytes = (int) map.get("jw.sqe.spout.kafka.fetchSizeBytes");
+ if(map.containsKey("jw.sqe.spout.kafka.filterReplays"))
+ options.filterReplays = (boolean) map.get("jw.sqe.spout.kafka.filterReplays");
+ if(map.containsKey("jw.sqe.spout.kafka.filterReplays.metadata.ttl"))
+ options.filterReplaysMetadataTtl = (long) map.get("jw.sqe.spout.kafka.filterReplays.metadata.ttl");
+ if(map.containsKey("jw.sqe.spout.kafka.maxOffsetBehind"))
+ options.maxOffsetBehind = (long) map.get("jw.sqe.spout.kafka.maxOffsetBehind");
+
+ return options;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/testing/FixedBatchSpoutOptions.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/testing/FixedBatchSpoutOptions.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/testing/FixedBatchSpoutOptions.java
new file mode 100644
index 0000000..defb4db
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/testing/FixedBatchSpoutOptions.java
@@ -0,0 +1,33 @@
+package com.jwplayer.sqe.language.stream.testing;
+
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Values;
+
+import java.io.Serializable;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+
+public class FixedBatchSpoutOptions implements Serializable {
+ public Fields fields;
+ public List<List<Object>> values;
+
+ @SuppressWarnings("unchecked")
+ public static FixedBatchSpoutOptions parse(Map map) {
+ FixedBatchSpoutOptions options = new FixedBatchSpoutOptions();
+
+ if(map.containsKey("jw.sqe.spout.fixed.fields")) options.fields =
+ new Fields((List) map.get("jw.sqe.spout.fixed.fields"));
+ options.values = new LinkedList<>();
+ if(map.containsKey("jw.sqe.spout.fixed.values")) {
+ List<List<Object>> rawValues = (List) map.get("jw.sqe.spout.fixed.values");
+
+ for(List<Object> rawValue: rawValues) {
+ options.values.add(new Values(rawValue.toArray()));
+ }
+ }
+
+ return options;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/testing/FixedBatchSpoutStreamAdapter.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/testing/FixedBatchSpoutStreamAdapter.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/testing/FixedBatchSpoutStreamAdapter.java
new file mode 100644
index 0000000..310763e
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/testing/FixedBatchSpoutStreamAdapter.java
@@ -0,0 +1,33 @@
+package com.jwplayer.sqe.language.stream.testing;
+
+import com.jwplayer.sqe.language.stream.StreamAdapter;
+import com.jwplayer.sqe.trident.StreamMetadata;
+import com.jwplayer.sqe.trident.function.AddField;
+import org.apache.storm.trident.Stream;
+import org.apache.storm.trident.TridentTopology;
+import org.apache.storm.trident.state.StateType;
+import org.apache.storm.trident.testing.FixedBatchSpout;
+import org.apache.storm.tuple.Fields;
+
+import java.util.List;
+
+
+public class FixedBatchSpoutStreamAdapter extends StreamAdapter {
+ FixedBatchSpoutOptions options;
+
+ public FixedBatchSpoutStreamAdapter(FixedBatchSpoutOptions options) {
+ this.options = options;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Stream makeStream(TridentTopology topology, String topologyName, String streamName, String objectName, StateType spoutType) {
+ List<Object>[] values = (List<Object>[]) options.values.toArray(new List[options.values.size()]);
+ FixedBatchSpout spout = new FixedBatchSpout(options.fields, options.values.size(), values);
+ Stream stream = topology.newStream(topologyName + "/" + streamName, spout);
+ long pid = StreamAdapter.createPid(topologyName, streamName, "");
+ StreamMetadata metadata = new StreamMetadata(pid, 0, 0L);
+ stream = stream.each(new Fields(), new AddField(metadata), new Fields(StreamAdapter.STREAM_METADATA_FIELD));
+ return stream;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/ListValuesCollector.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/ListValuesCollector.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/ListValuesCollector.java
new file mode 100644
index 0000000..eb638a3
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/ListValuesCollector.java
@@ -0,0 +1,19 @@
+package com.jwplayer.sqe.trident;
+
+import org.apache.storm.trident.operation.TridentCollector;
+
+import java.util.ArrayList;
+import java.util.List;
+
+
+public class ListValuesCollector implements TridentCollector {
+ public List<List<Object>> values = new ArrayList<>();
+
+ public void emit(List<Object> values) {
+ this.values.add(values);
+ }
+
+ public void reportError(Throwable throwable) {
+ throw new RuntimeException(throwable);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/SingleValuesCollector.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/SingleValuesCollector.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/SingleValuesCollector.java
new file mode 100644
index 0000000..99b6ad3
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/SingleValuesCollector.java
@@ -0,0 +1,18 @@
+package com.jwplayer.sqe.trident;
+
+import org.apache.storm.trident.operation.TridentCollector;
+
+import java.util.List;
+
+
+public class SingleValuesCollector implements TridentCollector {
+ public List<Object> values = null;
+
+ public void emit(List<Object> values) {
+ this.values = values;
+ }
+
+ public void reportError(Throwable throwable) {
+ throw new RuntimeException(throwable);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/StreamMetadata.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/StreamMetadata.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/StreamMetadata.java
new file mode 100644
index 0000000..913ea63
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/StreamMetadata.java
@@ -0,0 +1,43 @@
+package com.jwplayer.sqe.trident;
+
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
+import org.apache.storm.shade.org.apache.commons.lang.ArrayUtils;
+
+import java.io.Serializable;
+import java.util.Arrays;
+
+
+public class StreamMetadata implements Serializable {
+ public long pid;
+ public int partition;
+ public long offset;
+
+ public StreamMetadata(long pid, int partition, long offset) {
+ this.pid = pid;
+ this.partition = partition;
+ this.offset = offset;
+ }
+
+ public String getPidAndPartitionAsHex() {
+ return Long.toHexString(pid) + "-" + Integer.toHexString(partition);
+ }
+
+ public byte[] toBytes() {
+ return ArrayUtils.addAll(
+ Longs.toByteArray(pid),
+ ArrayUtils.addAll(Ints.toByteArray(partition), Longs.toByteArray(offset))
+ );
+ }
+
+ public static StreamMetadata parseBytes(byte[] bytes) {
+ Preconditions.checkArgument(bytes.length == 20, "Stream metadata bytes representation must contain exactly 20 bytes");
+
+ long pid = Longs.fromByteArray(Arrays.copyOfRange(bytes, 0, 8));
+ int partition = Ints.fromByteArray(Arrays.copyOfRange(bytes, 8, 12));
+ long offset = Longs.fromByteArray(Arrays.copyOfRange(bytes, 12, 20));
+
+ return new StreamMetadata(pid, partition, offset);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/ValueFilter.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/ValueFilter.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/ValueFilter.java
new file mode 100644
index 0000000..fad4651
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/ValueFilter.java
@@ -0,0 +1,11 @@
+package com.jwplayer.sqe.trident;
+
+import org.apache.storm.trident.operation.BaseFilter;
+import org.apache.storm.trident.tuple.TridentTuple;
+
+public class ValueFilter extends BaseFilter {
+ @Override
+ public boolean isKeep(TridentTuple tuple) {
+ return tuple.getBoolean(0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/CardinalityEstimatorAggregator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/CardinalityEstimatorAggregator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/CardinalityEstimatorAggregator.java
new file mode 100644
index 0000000..e106aa4
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/CardinalityEstimatorAggregator.java
@@ -0,0 +1,30 @@
+package com.jwplayer.sqe.trident.aggregator;
+
+import org.apache.storm.tuple.Values;
+import com.clearspring.analytics.stream.cardinality.ICardinality;
+import org.apache.storm.trident.operation.BaseAggregator;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+
+import java.io.IOException;
+
+
+/* This aggregator is for creating new cardinality estimators and efficiently adding fields you want
+ * to count. Once the object is created, you can use the associated CombinerAggregator for combing multiple
+ * objects into a single bitmap. */
+public abstract class CardinalityEstimatorAggregator extends BaseAggregator<ICardinality> {
+ @Override
+ public void aggregate(ICardinality cardinalityEstimator, TridentTuple tuple, TridentCollector collector) {
+ // Replicate SQL COUNT(DISTINCT) functionality to not count NULLs as a countable value
+ if(tuple.getValue(0) != null) cardinalityEstimator.offer(tuple.getValue(0));
+ }
+
+ @Override
+ public void complete(ICardinality cardinalityEstimator, TridentCollector collector) {
+ try {
+ collector.emit(new Values(cardinalityEstimator.getBytes()));
+ } catch(IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/CardinalityEstimatorCombinerAggregator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/CardinalityEstimatorCombinerAggregator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/CardinalityEstimatorCombinerAggregator.java
new file mode 100644
index 0000000..32da432
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/CardinalityEstimatorCombinerAggregator.java
@@ -0,0 +1,13 @@
+package com.jwplayer.sqe.trident.aggregator;
+
+import org.apache.storm.trident.operation.CombinerAggregator;
+import org.apache.storm.trident.tuple.TridentTuple;
+
+
+/* This aggregator is for taking existing cardinality estimator bitmaps/objects and aggregating/combining them. */
+public abstract class CardinalityEstimatorCombinerAggregator implements CombinerAggregator<byte[]> {
+ @Override
+ public byte[] init(TridentTuple tuple) {
+ return (byte[]) tuple.getValue(0);
+ }
+}
[08/11] storm git commit: SQE: Apply Apache license headers
Posted by pt...@apache.org.
SQE: Apply Apache license headers
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a667c75d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a667c75d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a667c75d
Branch: refs/heads/sqe_import
Commit: a667c75d025a5b7ba40afba5503e88c44a76ad91
Parents: 3654b8c
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Wed Sep 28 15:11:08 2016 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Wed Sep 28 15:11:08 2016 -0400
----------------------------------------------------------------------
external/storm-sqe/conf/multi-type-props.yaml | 15 +++++++++++++++
external/storm-sqe/conf/sample-conf.yaml | 15 +++++++++++++++
external/storm-sqe/pom.xml | 16 ++++++++++++++++
.../main/java/com/jwplayer/sqe/QueryEngine.java | 17 +++++++++++++++++
.../src/main/java/com/jwplayer/sqe/Topology.java | 17 +++++++++++++++++
.../jwplayer/sqe/language/clause/FromClause.java | 17 +++++++++++++++++
.../sqe/language/clause/InsertIntoClause.java | 17 +++++++++++++++++
.../jwplayer/sqe/language/clause/SelectClause.java | 17 +++++++++++++++++
.../jwplayer/sqe/language/command/BaseCommand.java | 17 +++++++++++++++++
.../jwplayer/sqe/language/command/CommandType.java | 17 +++++++++++++++++
.../sqe/language/command/CreateStream.java | 17 +++++++++++++++++
.../com/jwplayer/sqe/language/command/Query.java | 17 +++++++++++++++++
.../jwplayer/sqe/language/command/SetCommand.java | 17 +++++++++++++++++
.../sqe/language/expression/BaseExpression.java | 17 +++++++++++++++++
.../language/expression/ConstantExpression.java | 17 +++++++++++++++++
.../sqe/language/expression/ExpressionType.java | 17 +++++++++++++++++
.../sqe/language/expression/FieldExpression.java | 17 +++++++++++++++++
.../language/expression/FunctionExpression.java | 17 +++++++++++++++++
.../sqe/language/expression/FunctionType.java | 17 +++++++++++++++++
.../aggregation/AggregationExpression.java | 17 +++++++++++++++++
.../language/expression/aggregation/CreateHll.java | 17 +++++++++++++++++
.../expression/aggregation/CreateHllp.java | 17 +++++++++++++++++
.../sqe/language/expression/aggregation/Max.java | 17 +++++++++++++++++
.../sqe/language/expression/aggregation/Sum.java | 17 +++++++++++++++++
.../expression/transform/ArithmeticOperator.java | 17 +++++++++++++++++
.../transform/ArithmeticOperatorType.java | 17 +++++++++++++++++
.../transform/DivisionArithmeticOperator.java | 17 +++++++++++++++++
.../language/expression/transform/ExpandKeys.java | 17 +++++++++++++++++
.../expression/transform/ExpandValues.java | 17 +++++++++++++++++
.../language/expression/transform/FormatDate.java | 17 +++++++++++++++++
.../language/expression/transform/GetMapValue.java | 17 +++++++++++++++++
.../sqe/language/expression/transform/GetTime.java | 17 +++++++++++++++++
.../sqe/language/expression/transform/Hash.java | 17 +++++++++++++++++
.../sqe/language/expression/transform/If.java | 17 +++++++++++++++++
.../transform/ModulusArithmeticOperator.java | 17 +++++++++++++++++
.../MultiplicationArithmeticOperator.java | 17 +++++++++++++++++
.../language/expression/transform/ParseDate.java | 17 +++++++++++++++++
.../transform/PlusArithmeticOperator.java | 17 +++++++++++++++++
.../language/expression/transform/RoundDate.java | 17 +++++++++++++++++
.../transform/SubtractionArithmeticOperator.java | 17 +++++++++++++++++
.../expression/transform/TransformExpression.java | 17 +++++++++++++++++
.../transform/predicate/AndLogicalOperator.java | 17 +++++++++++++++++
.../transform/predicate/CompareNumbers.java | 17 +++++++++++++++++
.../expression/transform/predicate/Equals.java | 17 +++++++++++++++++
.../predicate/GreaterThanCompareNumbers.java | 17 +++++++++++++++++
.../predicate/GreaterThanEqualCompareNumbers.java | 17 +++++++++++++++++
.../expression/transform/predicate/In.java | 17 +++++++++++++++++
.../predicate/LessThanCompareNumbers.java | 17 +++++++++++++++++
.../predicate/LessThanEqualCompareNumbers.java | 17 +++++++++++++++++
.../transform/predicate/LogicalOperator.java | 17 +++++++++++++++++
.../transform/predicate/LogicalOperatorType.java | 17 +++++++++++++++++
.../predicate/NotEqualCompareNumbers.java | 17 +++++++++++++++++
.../transform/predicate/NotLogicalOperator.java | 17 +++++++++++++++++
.../transform/predicate/NumberComparisonType.java | 17 +++++++++++++++++
.../transform/predicate/OrLogicalOperator.java | 17 +++++++++++++++++
.../transform/predicate/PredicateExpression.java | 17 +++++++++++++++++
.../expression/transform/predicate/RLike.java | 17 +++++++++++++++++
.../transform/predicate/XorLogicalOperator.java | 17 +++++++++++++++++
.../sqe/language/serde/BaseDeserializer.java | 17 +++++++++++++++++
.../sqe/language/serde/IdentityDeserializer.java | 17 +++++++++++++++++
.../sqe/language/serde/avro/AvroDeserializer.java | 17 +++++++++++++++++
.../serde/avro/AvroDeserializerOptions.java | 17 +++++++++++++++++
.../jwplayer/sqe/language/state/StateAdapter.java | 17 +++++++++++++++++
.../sqe/language/state/StateOperationType.java | 17 +++++++++++++++++
.../state/kafka/JwTridentKafkaStateFactory.java | 17 +++++++++++++++++
.../language/state/kafka/KafkaStateAdapter.java | 17 +++++++++++++++++
.../language/state/kafka/KafkaStateOptions.java | 17 +++++++++++++++++
.../state/kafka/SourcePartitionPartitioner.java | 17 +++++++++++++++++
.../state/mongodb/MongoDBStateAdapter.java | 17 +++++++++++++++++
.../language/state/redis/RedisStateAdapter.java | 17 +++++++++++++++++
.../language/state/redis/RedisStateFactory.java | 17 +++++++++++++++++
.../language/state/redis/RedisStateOptions.java | 17 +++++++++++++++++
.../sqe/language/stream/InputStreamType.java | 17 +++++++++++++++++
.../sqe/language/stream/StreamAdapter.java | 17 +++++++++++++++++
.../sqe/language/stream/StreamContainer.java | 17 +++++++++++++++++
.../language/stream/kafka/KafkaStreamAdapter.java | 17 +++++++++++++++++
.../stream/kafka/KafkaStreamAdapterOptions.java | 17 +++++++++++++++++
.../stream/testing/FixedBatchSpoutOptions.java | 17 +++++++++++++++++
.../testing/FixedBatchSpoutStreamAdapter.java | 17 +++++++++++++++++
.../jwplayer/sqe/trident/ListValuesCollector.java | 17 +++++++++++++++++
.../sqe/trident/SingleValuesCollector.java | 17 +++++++++++++++++
.../com/jwplayer/sqe/trident/StreamMetadata.java | 17 +++++++++++++++++
.../java/com/jwplayer/sqe/trident/ValueFilter.java | 17 +++++++++++++++++
.../aggregator/CardinalityEstimatorAggregator.java | 17 +++++++++++++++++
.../CardinalityEstimatorCombinerAggregator.java | 17 +++++++++++++++++
.../trident/aggregator/GetMaximumAggregator.java | 17 +++++++++++++++++
.../sqe/trident/aggregator/HllAggregator.java | 17 +++++++++++++++++
.../trident/aggregator/HllCombinerAggregator.java | 17 +++++++++++++++++
.../sqe/trident/aggregator/HllpAggregator.java | 17 +++++++++++++++++
.../trident/aggregator/HllpCombinerAggregator.java | 17 +++++++++++++++++
.../jwplayer/sqe/trident/function/AddField.java | 17 +++++++++++++++++
.../sqe/trident/function/CompareNumbers.java | 17 +++++++++++++++++
.../sqe/trident/function/CompareObjects.java | 17 +++++++++++++++++
.../trident/function/ConvertMetadataToBytes.java | 17 +++++++++++++++++
.../function/EvaluateRegularExpression.java | 17 +++++++++++++++++
.../jwplayer/sqe/trident/function/ExpandKeys.java | 17 +++++++++++++++++
.../sqe/trident/function/ExpandValues.java | 17 +++++++++++++++++
.../jwplayer/sqe/trident/function/FormatDate.java | 17 +++++++++++++++++
.../trident/function/GetCardinalityEstimation.java | 17 +++++++++++++++++
.../com/jwplayer/sqe/trident/function/GetIf.java | 17 +++++++++++++++++
.../jwplayer/sqe/trident/function/GetMapValue.java | 17 +++++++++++++++++
.../com/jwplayer/sqe/trident/function/GetTime.java | 17 +++++++++++++++++
.../sqe/trident/function/GetTupleFromAvro.java | 17 +++++++++++++++++
.../com/jwplayer/sqe/trident/function/Hash.java | 17 +++++++++++++++++
.../com/jwplayer/sqe/trident/function/IsIn.java | 17 +++++++++++++++++
.../jwplayer/sqe/trident/function/MapField.java | 17 +++++++++++++++++
.../jwplayer/sqe/trident/function/ParseDate.java | 17 +++++++++++++++++
.../function/ProcessArithmeticOperator.java | 17 +++++++++++++++++
.../trident/function/ProcessLogicalOperator.java | 17 +++++++++++++++++
.../jwplayer/sqe/trident/function/RoundDate.java | 17 +++++++++++++++++
.../kafka/FilteredOpaqueTridentKafkaSpout.java | 17 +++++++++++++++++
.../FilteredTransactionalTridentKafkaSpout.java | 17 +++++++++++++++++
.../spout/kafka/FilteredTridentCollector.java | 17 +++++++++++++++++
.../spout/kafka/FilteredTridentKafkaEmitter.java | 17 +++++++++++++++++
.../sqe/trident/spout/kafka/SqeRawFullScheme.java | 17 +++++++++++++++++
.../sqe/trident/state/BaseGsonSerializer.java | 17 +++++++++++++++++
.../sqe/trident/state/GsonOpaqueSerializer.java | 17 +++++++++++++++++
.../trident/state/GsonTransactionalSerializer.java | 17 +++++++++++++++++
.../sqe/trident/state/mongodb/MongoDBState.java | 17 +++++++++++++++++
.../trident/state/mongodb/MongoDBStateOptions.java | 17 +++++++++++++++++
.../sqe/trident/state/redis/RedisKeyFactory.java | 17 +++++++++++++++++
.../java/com/jwplayer/sqe/util/FileHelper.java | 17 +++++++++++++++++
.../java/com/jwplayer/sqe/util/YamlConfig.java | 17 +++++++++++++++++
.../sqe/util/file/resource/ResourceFileSystem.java | 17 +++++++++++++++++
.../file/resource/ResourceFileSystemProvider.java | 17 +++++++++++++++++
.../sqe/util/file/resource/ResourcePath.java | 17 +++++++++++++++++
.../java/com/jwplayer/sqe/IntegrationTest.java | 17 +++++++++++++++++
.../jwplayer/sqe/KafkaReplaysIntegrationTest.java | 17 +++++++++++++++++
.../command/CommandDeserializationTest.java | 17 +++++++++++++++++
.../expression/ExpressionDeserializationTest.java | 17 +++++++++++++++++
.../expression/FunctionExpressionTest.java | 17 +++++++++++++++++
.../com/jwplayer/sqe/trident/GetTuplesFilter.java | 17 +++++++++++++++++
.../aggregator/GetMaximumAggregatorTest.java | 17 +++++++++++++++++
.../sqe/trident/aggregator/HllAggregatorTest.java | 17 +++++++++++++++++
.../aggregator/HllCombinerAggregatorTest.java | 17 +++++++++++++++++
.../sqe/trident/aggregator/HllpAggregatorTest.java | 17 +++++++++++++++++
.../aggregator/HllpCombinerAggregatorTest.java | 17 +++++++++++++++++
.../sqe/trident/function/AddFieldTest.java | 17 +++++++++++++++++
.../sqe/trident/function/CompareNumbersTest.java | 17 +++++++++++++++++
.../sqe/trident/function/CompareObjectsTest.java | 17 +++++++++++++++++
.../function/EvaluateRegularExpressionTest.java | 17 +++++++++++++++++
.../sqe/trident/function/ExpandKeysTest.java | 17 +++++++++++++++++
.../sqe/trident/function/ExpandValuesTest.java | 17 +++++++++++++++++
.../sqe/trident/function/FormatDateTest.java | 17 +++++++++++++++++
.../function/GetCardinalityEstimationTest.java | 17 +++++++++++++++++
.../jwplayer/sqe/trident/function/GetIfTest.java | 17 +++++++++++++++++
.../sqe/trident/function/GetMapValueTest.java | 17 +++++++++++++++++
.../jwplayer/sqe/trident/function/GetTimeTest.java | 17 +++++++++++++++++
.../jwplayer/sqe/trident/function/HashTest.java | 17 +++++++++++++++++
.../jwplayer/sqe/trident/function/IsInTest.java | 17 +++++++++++++++++
.../sqe/trident/function/ParseDateTest.java | 17 +++++++++++++++++
.../function/ProcessArithmeticOperatorTest.java | 17 +++++++++++++++++
.../function/ProcessLogicalOperatorTest.java | 17 +++++++++++++++++
.../sqe/trident/function/RoundDateTest.java | 17 +++++++++++++++++
.../spout/kafka/FilteredTridentCollectorTest.java | 17 +++++++++++++++++
.../trident/spout/kafka/SqeRawFullSchemeTest.java | 17 +++++++++++++++++
.../java/com/jwplayer/sqe/util/FileHelperTest.java | 17 +++++++++++++++++
.../com/jwplayer/sqe/util/KafkaTestServer.java | 17 +++++++++++++++++
.../java/com/jwplayer/sqe/util/YamlConfigTest.java | 17 +++++++++++++++++
.../com/jwplayer/sqe/util/ZookeeperTestServer.java | 17 +++++++++++++++++
160 files changed, 2715 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/conf/multi-type-props.yaml
----------------------------------------------------------------------
diff --git a/external/storm-sqe/conf/multi-type-props.yaml b/external/storm-sqe/conf/multi-type-props.yaml
index 1a332d4..a30d6af 100644
--- a/external/storm-sqe/conf/multi-type-props.yaml
+++ b/external/storm-sqe/conf/multi-type-props.yaml
@@ -1,3 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
none: null
bool: true
int: 42
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/conf/sample-conf.yaml
----------------------------------------------------------------------
diff --git a/external/storm-sqe/conf/sample-conf.yaml b/external/storm-sqe/conf/sample-conf.yaml
index 67dcfbc..0805a08 100644
--- a/external/storm-sqe/conf/sample-conf.yaml
+++ b/external/storm-sqe/conf/sample-conf.yaml
@@ -1,3 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
Storm:
topology.workers: 1
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-sqe/pom.xml b/external/storm-sqe/pom.xml
index 550e16c..d5b7f4c 100644
--- a/external/storm-sqe/pom.xml
+++ b/external/storm-sqe/pom.xml
@@ -1,4 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/QueryEngine.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/QueryEngine.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/QueryEngine.java
index 9358a99..8de1d5a 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/QueryEngine.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/QueryEngine.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/Topology.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/Topology.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/Topology.java
index f508169..29b3e3a 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/Topology.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/Topology.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/FromClause.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/FromClause.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/FromClause.java
index e076e71..d1fc13c 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/FromClause.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/FromClause.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.clause;
public class FromClause {
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/InsertIntoClause.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/InsertIntoClause.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/InsertIntoClause.java
index e7c757e..4589756 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/InsertIntoClause.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/InsertIntoClause.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.clause;
import org.apache.storm.trident.state.StateType;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/SelectClause.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/SelectClause.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/SelectClause.java
index fe28ee1..dddfd51 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/SelectClause.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/SelectClause.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.clause;
import com.jwplayer.sqe.language.expression.BaseExpression;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/BaseCommand.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/BaseCommand.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/BaseCommand.java
index c29e7b5..1d52c0e 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/BaseCommand.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/BaseCommand.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.command;
import com.google.gson.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/CommandType.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/CommandType.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/CommandType.java
index 2e898fd..83ee954 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/CommandType.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/CommandType.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.command;
public enum CommandType {
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/CreateStream.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/CreateStream.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/CreateStream.java
index dddeb3b..8ddacf8 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/CreateStream.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/CreateStream.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.command;
import org.apache.storm.trident.state.StateType;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/Query.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/Query.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/Query.java
index 78879aa..22e9f6d 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/Query.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/Query.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.command;
import com.google.gson.GsonBuilder;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/SetCommand.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/SetCommand.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/SetCommand.java
index 8aba0ec..bd512f8 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/SetCommand.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/SetCommand.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.command;
public class SetCommand extends BaseCommand {
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/BaseExpression.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/BaseExpression.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/BaseExpression.java
index ce7ac47..1aa4eca 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/BaseExpression.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/BaseExpression.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression;
import com.google.gson.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/ConstantExpression.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/ConstantExpression.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/ConstantExpression.java
index c734717..da01064 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/ConstantExpression.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/ConstantExpression.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/ExpressionType.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/ExpressionType.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/ExpressionType.java
index d625e09..b20b252 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/ExpressionType.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/ExpressionType.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression;
public enum ExpressionType {
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FieldExpression.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FieldExpression.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FieldExpression.java
index e171256..1d16801 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FieldExpression.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FieldExpression.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FunctionExpression.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FunctionExpression.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FunctionExpression.java
index 3e5d0c8..eedbb50 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FunctionExpression.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FunctionExpression.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression;
import java.util.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FunctionType.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FunctionType.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FunctionType.java
index 331b0c2..0ae72df 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FunctionType.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FunctionType.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression;
public enum FunctionType {
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/AggregationExpression.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/AggregationExpression.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/AggregationExpression.java
index d1c1719..9c6685a 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/AggregationExpression.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/AggregationExpression.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.aggregation;
import com.jwplayer.sqe.language.expression.BaseExpression;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/CreateHll.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/CreateHll.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/CreateHll.java
index 14089ef..0e4f151 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/CreateHll.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/CreateHll.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.aggregation;
import com.jwplayer.sqe.language.expression.BaseExpression;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/CreateHllp.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/CreateHllp.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/CreateHllp.java
index e495af0..0b593c8 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/CreateHllp.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/CreateHllp.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.aggregation;
import com.jwplayer.sqe.trident.aggregator.HllpAggregator;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/Max.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/Max.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/Max.java
index d9a867b..857c65e 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/Max.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/Max.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.aggregation;
import com.jwplayer.sqe.trident.aggregator.GetMaximumAggregator;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/Sum.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/Sum.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/Sum.java
index d4f5748..f6bccaa 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/Sum.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/Sum.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.aggregation;
import org.apache.storm.trident.fluent.ChainedFullAggregatorDeclarer;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ArithmeticOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ArithmeticOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ArithmeticOperator.java
index c11a23f..59ca8d3 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ArithmeticOperator.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ArithmeticOperator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform;
import com.jwplayer.sqe.trident.function.ProcessArithmeticOperator;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ArithmeticOperatorType.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ArithmeticOperatorType.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ArithmeticOperatorType.java
index 7f358d4..77cf9a7 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ArithmeticOperatorType.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ArithmeticOperatorType.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform;
public enum ArithmeticOperatorType {
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/DivisionArithmeticOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/DivisionArithmeticOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/DivisionArithmeticOperator.java
index 1dda56c..b675785 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/DivisionArithmeticOperator.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/DivisionArithmeticOperator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ExpandKeys.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ExpandKeys.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ExpandKeys.java
index 1e08781..d4180ba 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ExpandKeys.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ExpandKeys.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform;
import org.apache.storm.trident.Stream;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ExpandValues.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ExpandValues.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ExpandValues.java
index 9c3d331..032df4a 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ExpandValues.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ExpandValues.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform;
import org.apache.storm.trident.Stream;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/FormatDate.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/FormatDate.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/FormatDate.java
index dace3f5..c1bb48b 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/FormatDate.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/FormatDate.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform;
import org.apache.storm.trident.Stream;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/GetMapValue.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/GetMapValue.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/GetMapValue.java
index 2d50425..819beaa 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/GetMapValue.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/GetMapValue.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform;
import org.apache.storm.trident.Stream;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/GetTime.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/GetTime.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/GetTime.java
index 747a71a..f504c08 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/GetTime.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/GetTime.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform;
import org.apache.storm.trident.Stream;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/Hash.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/Hash.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/Hash.java
index f354a5f..92cfe64 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/Hash.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/Hash.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform;
import org.apache.storm.trident.Stream;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/If.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/If.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/If.java
index fe61093..f90daad 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/If.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/If.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform;
import com.jwplayer.sqe.trident.function.GetIf;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ModulusArithmeticOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ModulusArithmeticOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ModulusArithmeticOperator.java
index 86b077f..59077c9 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ModulusArithmeticOperator.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ModulusArithmeticOperator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/MultiplicationArithmeticOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/MultiplicationArithmeticOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/MultiplicationArithmeticOperator.java
index 232cedb..b513070 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/MultiplicationArithmeticOperator.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/MultiplicationArithmeticOperator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ParseDate.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ParseDate.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ParseDate.java
index be74931..d098981 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ParseDate.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ParseDate.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform;
import org.apache.storm.trident.Stream;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/PlusArithmeticOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/PlusArithmeticOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/PlusArithmeticOperator.java
index 7268bea..a7c355a 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/PlusArithmeticOperator.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/PlusArithmeticOperator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/RoundDate.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/RoundDate.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/RoundDate.java
index 5f4c02b..e237f41 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/RoundDate.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/RoundDate.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform;
import org.apache.storm.trident.Stream;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/SubtractionArithmeticOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/SubtractionArithmeticOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/SubtractionArithmeticOperator.java
index c2d0c95..09030b3 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/SubtractionArithmeticOperator.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/SubtractionArithmeticOperator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/TransformExpression.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/TransformExpression.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/TransformExpression.java
index 2d4bb8c..c4211d3 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/TransformExpression.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/TransformExpression.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform;
import com.jwplayer.sqe.language.expression.FunctionExpression;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/AndLogicalOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/AndLogicalOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/AndLogicalOperator.java
index b8f82fa..bb4b338 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/AndLogicalOperator.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/AndLogicalOperator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform.predicate;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/CompareNumbers.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/CompareNumbers.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/CompareNumbers.java
index d7e186b..4419fa8 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/CompareNumbers.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/CompareNumbers.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform.predicate;
import org.apache.storm.trident.Stream;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/Equals.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/Equals.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/Equals.java
index 7b6c123..60def5e 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/Equals.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/Equals.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform.predicate;
import com.jwplayer.sqe.trident.function.CompareObjects;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/GreaterThanCompareNumbers.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/GreaterThanCompareNumbers.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/GreaterThanCompareNumbers.java
index 8431754..858dcf6 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/GreaterThanCompareNumbers.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/GreaterThanCompareNumbers.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform.predicate;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/GreaterThanEqualCompareNumbers.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/GreaterThanEqualCompareNumbers.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/GreaterThanEqualCompareNumbers.java
index 7663e44..189f122 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/GreaterThanEqualCompareNumbers.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/GreaterThanEqualCompareNumbers.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform.predicate;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/In.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/In.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/In.java
index 4f9f771..c583401 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/In.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/In.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform.predicate;
import com.jwplayer.sqe.trident.function.IsIn;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LessThanCompareNumbers.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LessThanCompareNumbers.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LessThanCompareNumbers.java
index d1e7223..9b62612 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LessThanCompareNumbers.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LessThanCompareNumbers.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform.predicate;
[09/11] storm git commit: SQE: Integrate SQE into main build
Posted by pt...@apache.org.
SQE: Integrate SQE into main build
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b28921d3
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b28921d3
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b28921d3
Branch: refs/heads/sqe_import
Commit: b28921d36de688cefc0e99828a4d0ded4b510412
Parents: a667c75
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Wed Sep 28 15:40:58 2016 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Wed Sep 28 15:40:58 2016 -0400
----------------------------------------------------------------------
external/storm-sqe/pom.xml | 44 ++++++++++++-----------------------------
pom.xml | 1 +
2 files changed, 14 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b28921d3/external/storm-sqe/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-sqe/pom.xml b/external/storm-sqe/pom.xml
index d5b7f4c..4e9a04b 100644
--- a/external/storm-sqe/pom.xml
+++ b/external/storm-sqe/pom.xml
@@ -19,21 +19,15 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>com.jwplayer</groupId>
- <artifactId>sqe</artifactId>
- <version>${build.number}</version>
- <repositories>
- <repository>
- <id>central</id>
- <name>Maven Central</name>
- <url>http://repo1.maven.org/maven2</url>
- </repository>
- </repositories>
- <properties>
- <maven.compiler.target>1.7</maven.compiler.target>
- <maven.compiler.source>1.7</maven.compiler.source>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
+ <parent>
+ <artifactId>storm</artifactId>
+ <groupId>org.apache.storm</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>storm-sqe</artifactId>
+
<dependencies>
<dependency>
<groupId>com.googlecode.json-simple</groupId>
@@ -72,23 +66,23 @@
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-redis</artifactId>
- <version>1.0.2</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-kafka</artifactId>
- <version>1.0.2</version>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
- <version>1.0.2</version>
+ <version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-starter</artifactId>
- <version>1.0.2</version>
+ <version>${project.version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
@@ -190,16 +184,4 @@
<scope>test</scope>
</dependency>
</dependencies>
- <profiles>
- <profile>
- <id>default</id>
- <properties>
- <env>default</env>
- <build.number>1.0-SNAPSHOT</build.number>
- </properties>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
- </profile>
- </profiles>
</project>
http://git-wip-us.apache.org/repos/asf/storm/blob/b28921d3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7bc566d..713e17a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -319,6 +319,7 @@
<module>external/storm-kinesis</module>
<module>external/storm-druid</module>
<module>external/storm-submit-tools</module>
+ <module>external/storm-sqe</module>
<!-- examples -->
<module>examples/storm-starter</module>
[05/11] storm git commit: initial SQE import
Posted by pt...@apache.org.
initial SQE import
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3654b8ca
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3654b8ca
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3654b8ca
Branch: refs/heads/sqe_import
Commit: 3654b8caf161768255b77a210d0234a373e86512
Parents: f3e4dd3
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Wed Sep 28 14:27:53 2016 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Wed Sep 28 14:27:53 2016 -0400
----------------------------------------------------------------------
external/storm-sqe/README.md | 10 +
external/storm-sqe/conf/multi-type-props.yaml | 6 +
external/storm-sqe/conf/sample-conf.yaml | 8 +
external/storm-sqe/pom.xml | 189 ++++++++
.../main/java/com/jwplayer/sqe/QueryEngine.java | 480 +++++++++++++++++++
.../main/java/com/jwplayer/sqe/Topology.java | 126 +++++
.../sqe/language/clause/FromClause.java | 9 +
.../sqe/language/clause/InsertIntoClause.java | 23 +
.../sqe/language/clause/SelectClause.java | 13 +
.../sqe/language/command/BaseCommand.java | 57 +++
.../sqe/language/command/CommandType.java | 7 +
.../sqe/language/command/CreateStream.java | 20 +
.../jwplayer/sqe/language/command/Query.java | 33 ++
.../sqe/language/command/SetCommand.java | 11 +
.../sqe/language/expression/BaseExpression.java | 104 ++++
.../language/expression/ConstantExpression.java | 35 ++
.../sqe/language/expression/ExpressionType.java | 7 +
.../language/expression/FieldExpression.java | 30 ++
.../language/expression/FunctionExpression.java | 119 +++++
.../sqe/language/expression/FunctionType.java | 6 +
.../aggregation/AggregationExpression.java | 55 +++
.../expression/aggregation/CreateHll.java | 56 +++
.../expression/aggregation/CreateHllp.java | 35 ++
.../language/expression/aggregation/Max.java | 36 ++
.../language/expression/aggregation/Sum.java | 33 ++
.../transform/ArithmeticOperator.java | 19 +
.../transform/ArithmeticOperatorType.java | 9 +
.../transform/DivisionArithmeticOperator.java | 13 +
.../expression/transform/ExpandKeys.java | 19 +
.../expression/transform/ExpandValues.java | 19 +
.../expression/transform/FormatDate.java | 19 +
.../expression/transform/GetMapValue.java | 19 +
.../language/expression/transform/GetTime.java | 19 +
.../sqe/language/expression/transform/Hash.java | 19 +
.../sqe/language/expression/transform/If.java | 20 +
.../transform/ModulusArithmeticOperator.java | 13 +
.../MultiplicationArithmeticOperator.java | 13 +
.../expression/transform/ParseDate.java | 19 +
.../transform/PlusArithmeticOperator.java | 13 +
.../expression/transform/RoundDate.java | 19 +
.../SubtractionArithmeticOperator.java | 13 +
.../transform/TransformExpression.java | 18 +
.../transform/predicate/AndLogicalOperator.java | 13 +
.../transform/predicate/CompareNumbers.java | 21 +
.../expression/transform/predicate/Equals.java | 20 +
.../predicate/GreaterThanCompareNumbers.java | 13 +
.../GreaterThanEqualCompareNumbers.java | 13 +
.../expression/transform/predicate/In.java | 20 +
.../predicate/LessThanCompareNumbers.java | 13 +
.../predicate/LessThanEqualCompareNumbers.java | 13 +
.../transform/predicate/LogicalOperator.java | 19 +
.../predicate/LogicalOperatorType.java | 8 +
.../predicate/NotEqualCompareNumbers.java | 13 +
.../transform/predicate/NotLogicalOperator.java | 13 +
.../predicate/NumberComparisonType.java | 10 +
.../transform/predicate/OrLogicalOperator.java | 13 +
.../predicate/PredicateExpression.java | 9 +
.../expression/transform/predicate/RLike.java | 20 +
.../transform/predicate/XorLogicalOperator.java | 13 +
.../sqe/language/serde/BaseDeserializer.java | 25 +
.../language/serde/IdentityDeserializer.java | 16 +
.../language/serde/avro/AvroDeserializer.java | 21 +
.../serde/avro/AvroDeserializerOptions.java | 16 +
.../sqe/language/state/StateAdapter.java | 41 ++
.../sqe/language/state/StateOperationType.java | 6 +
.../state/kafka/JwTridentKafkaStateFactory.java | 75 +++
.../language/state/kafka/KafkaStateAdapter.java | 76 +++
.../language/state/kafka/KafkaStateOptions.java | 51 ++
.../state/kafka/SourcePartitionPartitioner.java | 34 ++
.../state/mongodb/MongoDBStateAdapter.java | 31 ++
.../language/state/redis/RedisStateAdapter.java | 142 ++++++
.../language/state/redis/RedisStateFactory.java | 29 ++
.../language/state/redis/RedisStateOptions.java | 43 ++
.../sqe/language/stream/InputStreamType.java | 6 +
.../sqe/language/stream/StreamAdapter.java | 36 ++
.../sqe/language/stream/StreamContainer.java | 15 +
.../stream/kafka/KafkaStreamAdapter.java | 52 ++
.../stream/kafka/KafkaStreamAdapterOptions.java | 54 +++
.../stream/testing/FixedBatchSpoutOptions.java | 33 ++
.../testing/FixedBatchSpoutStreamAdapter.java | 33 ++
.../sqe/trident/ListValuesCollector.java | 19 +
.../sqe/trident/SingleValuesCollector.java | 18 +
.../jwplayer/sqe/trident/StreamMetadata.java | 43 ++
.../com/jwplayer/sqe/trident/ValueFilter.java | 11 +
.../CardinalityEstimatorAggregator.java | 30 ++
.../CardinalityEstimatorCombinerAggregator.java | 13 +
.../aggregator/GetMaximumAggregator.java | 73 +++
.../sqe/trident/aggregator/HllAggregator.java | 17 +
.../aggregator/HllCombinerAggregator.java | 36 ++
.../sqe/trident/aggregator/HllpAggregator.java | 18 +
.../aggregator/HllpCombinerAggregator.java | 35 ++
.../jwplayer/sqe/trident/function/AddField.java | 19 +
.../sqe/trident/function/CompareNumbers.java | 45 ++
.../sqe/trident/function/CompareObjects.java | 25 +
.../function/ConvertMetadataToBytes.java | 17 +
.../function/EvaluateRegularExpression.java | 30 ++
.../sqe/trident/function/ExpandKeys.java | 29 ++
.../sqe/trident/function/ExpandValues.java | 30 ++
.../sqe/trident/function/FormatDate.java | 26 +
.../function/GetCardinalityEstimation.java | 33 ++
.../jwplayer/sqe/trident/function/GetIf.java | 17 +
.../sqe/trident/function/GetMapValue.java | 24 +
.../jwplayer/sqe/trident/function/GetTime.java | 35 ++
.../sqe/trident/function/GetTupleFromAvro.java | 70 +++
.../com/jwplayer/sqe/trident/function/Hash.java | 37 ++
.../com/jwplayer/sqe/trident/function/IsIn.java | 31 ++
.../jwplayer/sqe/trident/function/MapField.java | 14 +
.../sqe/trident/function/ParseDate.java | 26 +
.../function/ProcessArithmeticOperator.java | 46 ++
.../function/ProcessLogicalOperator.java | 45 ++
.../sqe/trident/function/RoundDate.java | 44 ++
.../kafka/FilteredOpaqueTridentKafkaSpout.java | 45 ++
.../FilteredTransactionalTridentKafkaSpout.java | 46 ++
.../spout/kafka/FilteredTridentCollector.java | 88 ++++
.../kafka/FilteredTridentKafkaEmitter.java | 101 ++++
.../trident/spout/kafka/SqeRawFullScheme.java | 38 ++
.../sqe/trident/state/BaseGsonSerializer.java | 74 +++
.../sqe/trident/state/GsonOpaqueSerializer.java | 46 ++
.../state/GsonTransactionalSerializer.java | 38 ++
.../sqe/trident/state/mongodb/MongoDBState.java | 228 +++++++++
.../state/mongodb/MongoDBStateOptions.java | 35 ++
.../trident/state/redis/RedisKeyFactory.java | 47 ++
.../java/com/jwplayer/sqe/util/FileHelper.java | 44 ++
.../java/com/jwplayer/sqe/util/YamlConfig.java | 149 ++++++
.../util/file/resource/ResourceFileSystem.java | 89 ++++
.../resource/ResourceFileSystemProvider.java | 108 +++++
.../sqe/util/file/resource/ResourcePath.java | 149 ++++++
...r.sqe.language.expression.FunctionExpression | 30 ++
.../java.nio.file.spi.FileSystemProvider | 1 +
.../java/com/jwplayer/sqe/IntegrationTest.java | 71 +++
.../sqe/KafkaReplaysIntegrationTest.java | 131 +++++
.../command/CommandDeserializationTest.java | 49 ++
.../ExpressionDeserializationTest.java | 103 ++++
.../expression/FunctionExpressionTest.java | 46 ++
.../jwplayer/sqe/trident/GetTuplesFilter.java | 23 +
.../aggregator/GetMaximumAggregatorTest.java | 64 +++
.../trident/aggregator/HllAggregatorTest.java | 43 ++
.../aggregator/HllCombinerAggregatorTest.java | 45 ++
.../trident/aggregator/HllpAggregatorTest.java | 43 ++
.../aggregator/HllpCombinerAggregatorTest.java | 45 ++
.../sqe/trident/function/AddFieldTest.java | 52 ++
.../trident/function/CompareNumbersTest.java | 76 +++
.../trident/function/CompareObjectsTest.java | 81 ++++
.../function/EvaluateRegularExpressionTest.java | 31 ++
.../sqe/trident/function/ExpandKeysTest.java | 37 ++
.../sqe/trident/function/ExpandValuesTest.java | 37 ++
.../sqe/trident/function/FormatDateTest.java | 27 ++
.../function/GetCardinalityEstimationTest.java | 48 ++
.../sqe/trident/function/GetIfTest.java | 31 ++
.../sqe/trident/function/GetMapValueTest.java | 36 ++
.../sqe/trident/function/GetTimeTest.java | 34 ++
.../jwplayer/sqe/trident/function/HashTest.java | 24 +
.../jwplayer/sqe/trident/function/IsInTest.java | 40 ++
.../sqe/trident/function/ParseDateTest.java | 27 ++
.../function/ProcessArithmeticOperatorTest.java | 255 ++++++++++
.../function/ProcessLogicalOperatorTest.java | 121 +++++
.../sqe/trident/function/RoundDateTest.java | 62 +++
.../kafka/FilteredTridentCollectorTest.java | 57 +++
.../spout/kafka/SqeRawFullSchemeTest.java | 27 ++
.../com/jwplayer/sqe/util/FileHelperTest.java | 42 ++
.../com/jwplayer/sqe/util/KafkaTestServer.java | 107 +++++
.../com/jwplayer/sqe/util/YamlConfigTest.java | 65 +++
.../jwplayer/sqe/util/ZookeeperTestServer.java | 48 ++
.../queries/command-deserialization.json | 71 +++
.../test/resources/queries/kafka-replays.json | 51 ++
.../resources/queries/test-complex-query.json | 66 +++
.../src/test/resources/sample-resource.txt | 5 +
167 files changed, 7403 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/README.md
----------------------------------------------------------------------
diff --git a/external/storm-sqe/README.md b/external/storm-sqe/README.md
new file mode 100644
index 0000000..44e0fa8
--- /dev/null
+++ b/external/storm-sqe/README.md
@@ -0,0 +1,10 @@
+SQE - Streaming Query Engine
+===================
+
+## What is SQE?
+
+SQE is a query engine written using Storm's Trident framework that takes a set of SQL-like (inspired?) commands, including queries, and runs them against one or more input streams. By using Trident, input streams are processed in micro-batches with both good latency and high throughput while guaranteeing "exactly-once" processing. Results can be returned through a list of output streams or SQE can handle persisting to a data store directly using one of its supported Trident states. SQE is designed to make it easy to query against large streams of data with good performance for many different use cases.
+
+There are two external jars included in the repo, both forks of Storm external projects. The forked code can be found here: https://github.com/jwplayer/storm/tree/1.x-redis-hash
+
+https://github.com/jwplayer/sqe/wiki
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/conf/multi-type-props.yaml
----------------------------------------------------------------------
diff --git a/external/storm-sqe/conf/multi-type-props.yaml b/external/storm-sqe/conf/multi-type-props.yaml
new file mode 100644
index 0000000..1a332d4
--- /dev/null
+++ b/external/storm-sqe/conf/multi-type-props.yaml
@@ -0,0 +1,6 @@
+none: null
+bool: true
+int: 42
+float: 3.14159
+list: [LITE, RES_ACID, SUS_DEXT]
+map: {hp: 13, sp: 5, name: SusyTheBarbarian}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/conf/sample-conf.yaml
----------------------------------------------------------------------
diff --git a/external/storm-sqe/conf/sample-conf.yaml b/external/storm-sqe/conf/sample-conf.yaml
new file mode 100644
index 0000000..67dcfbc
--- /dev/null
+++ b/external/storm-sqe/conf/sample-conf.yaml
@@ -0,0 +1,8 @@
+Storm:
+ topology.workers: 1
+
+SQE:
+ jw.sqe.parallelism.hint: 1
+ jw.sqe.state.redis.database: 1
+ jw.sqe.spout.kafka.bufferSizeBytes: 65536 # This is less than the default for testing purposes
+ jw.sqe.spout.kafka.fetchSizeBytes: 65536 # This is less than the default for testing purposes
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-sqe/pom.xml b/external/storm-sqe/pom.xml
new file mode 100644
index 0000000..550e16c
--- /dev/null
+++ b/external/storm-sqe/pom.xml
@@ -0,0 +1,189 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>com.jwplayer</groupId>
+ <artifactId>sqe</artifactId>
+ <version>${build.number}</version>
+ <repositories>
+ <repository>
+ <id>central</id>
+ <name>Maven Central</name>
+ <url>http://repo1.maven.org/maven2</url>
+ </repository>
+ </repositories>
+ <properties>
+ <maven.compiler.target>1.7</maven.compiler.target>
+ <maven.compiler.source>1.7</maven.compiler.source>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>com.googlecode.json-simple</groupId>
+ <artifactId>json-simple</artifactId>
+ <version>1.1.1</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>1.8</version>
+ </dependency>
+ <dependency>
+ <groupId>net.sf.jopt-simple</groupId>
+ <artifactId>jopt-simple</artifactId>
+ <version>4.6</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.11</artifactId>
+ <version>0.9.0.1</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-redis</artifactId>
+ <version>1.0.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-kafka</artifactId>
+ <version>1.0.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ <version>1.0.2</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-starter</artifactId>
+ <version>1.0.2</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.clearspring.analytics</groupId>
+ <artifactId>stream</artifactId>
+ <version>2.7.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.3.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.mongodb</groupId>
+ <artifactId>mongodb-driver</artifactId>
+ <version>3.0.3</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.11</version>
+ </dependency>
+ <dependency>
+ <groupId>redis.clients</groupId>
+ <artifactId>jedis</artifactId>
+ <version>2.7.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <version>1.7.6</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.4</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>18.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ <version>1.16</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>2.10.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ <version>2.10.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-test</artifactId>
+ <version>2.10.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <profiles>
+ <profile>
+ <id>default</id>
+ <properties>
+ <env>default</env>
+ <build.number>1.0-SNAPSHOT</build.number>
+ </properties>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ </profile>
+ </profiles>
+</project>
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/QueryEngine.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/QueryEngine.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/QueryEngine.java
new file mode 100644
index 0000000..9358a99
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/QueryEngine.java
@@ -0,0 +1,480 @@
+package com.jwplayer.sqe;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import com.jwplayer.sqe.language.clause.InsertIntoClause;
+import com.jwplayer.sqe.language.command.BaseCommand;
+import com.jwplayer.sqe.language.command.CommandType;
+import com.jwplayer.sqe.language.command.CreateStream;
+import com.jwplayer.sqe.language.command.Query;
+import com.jwplayer.sqe.language.command.SetCommand;
+import com.jwplayer.sqe.language.expression.BaseExpression;
+import com.jwplayer.sqe.language.expression.ConstantExpression;
+import com.jwplayer.sqe.language.expression.ExpressionType;
+import com.jwplayer.sqe.language.expression.FieldExpression;
+import com.jwplayer.sqe.language.expression.FunctionExpression;
+import com.jwplayer.sqe.language.expression.FunctionType;
+import com.jwplayer.sqe.language.expression.aggregation.AggregationExpression;
+import com.jwplayer.sqe.language.expression.transform.TransformExpression;
+import com.jwplayer.sqe.language.serde.BaseDeserializer;
+import com.jwplayer.sqe.language.state.StateAdapter;
+import com.jwplayer.sqe.language.state.StateOperationType;
+import com.jwplayer.sqe.language.stream.InputStreamType;
+import com.jwplayer.sqe.language.stream.StreamAdapter;
+import com.jwplayer.sqe.language.stream.StreamContainer;
+import com.jwplayer.sqe.trident.ValueFilter;
+import com.jwplayer.sqe.trident.function.AddField;
+import com.jwplayer.sqe.trident.function.GetTupleFromAvro;
+import com.jwplayer.sqe.trident.function.MapField;
+import org.apache.storm.trident.Stream;
+import org.apache.storm.trident.TridentTopology;
+import org.apache.storm.trident.fluent.ChainedFullAggregatorDeclarer;
+import org.apache.storm.trident.fluent.ChainedPartitionAggregatorDeclarer;
+import org.apache.storm.trident.fluent.GroupedStream;
+import org.apache.storm.trident.state.StateFactory;
+import org.apache.storm.tuple.Fields;
+
+
+public class QueryEngine {
+ private Map<String, BaseCommand> commands = new LinkedHashMap<>();
+ private Map<String, StreamContainer> inputStreams = new HashMap<>();
+ private Map optionMap;
+ private String topologyName;
+
+ public QueryEngine(String topologyName, Map optionMap) {
+ this.optionMap = optionMap;
+ this.topologyName = topologyName;
+ }
+
+ public static class Options implements Serializable {
+ public int parallelismHint = 1;
+
+ public static Options parse(Map map) {
+ Options options = new Options();
+
+ if(map.containsKey("jw.sqe.parallelism.hint")) options.parallelismHint = (int) map.get("jw.sqe.parallelism.hint");
+
+ return options;
+ }
+ }
+
+ // TODO: This is for backwards compatibility, remove later
+ public Map<String, Stream> build() {
+ return build(null);
+ }
+
+ /*
+ This builds the necessary components (each, aggregate, etc) onto the input streams to compute the queries.
+ While I'm trying to keep this flexible for future expansion, there are some pretty big assumptions in play:
+
+ - TODO: If more than one query is supplied, the query engine will create a big parent query that is the
+ union of all of the fields and aggregations of the child queries. This *should* be a huge performance
+ boost for the first few sets of queries we want to do, but the query engine is not smart here and can be
+ greatly improved in the future. The parent query will also be generated in a brute force way: the query engine
+ won't try to determine if it can combine or choose a single transform to meet multiple transforms downstream.
+ All transforms are promoted to the parent and no longer appear in the children queries. Aggregations will be
+ run in both the parent and child queries.
+ */
+ @SuppressWarnings("unchecked")
+ public Map<String, Stream> build(TridentTopology topology) {
+ Options options = Options.parse(optionMap);
+ if(commands.size() == 0) throw new RuntimeException("There are no registered commands");
+
+ // Determine needed fields
+
+ Map<String, Set<String>> requiredFieldsPerStreamMap = new HashMap<>();
+
+ for(Map.Entry<String, BaseCommand> command: commands.entrySet()) {
+ // Only check Query commands
+ if(command.getValue().getCommandType() == CommandType.Query) {
+ Query query = (Query) command.getValue();
+ Set<String> requiredFieldsForQuery = new HashSet<>();
+ requiredFieldsForQuery.add(StreamAdapter.STREAM_METADATA_FIELD);
+ requiredFieldsForQuery.addAll(getFields(query.select.expressions));
+
+ if (query.where != null)
+ requiredFieldsForQuery.addAll(getFields(query.where));
+
+ if (requiredFieldsPerStreamMap.containsKey(query.from.objectName)){
+ Set<String> FieldsFromMapForStream = requiredFieldsPerStreamMap.get(query.from.objectName);
+ requiredFieldsForQuery.addAll(FieldsFromMapForStream);
+ }
+
+ requiredFieldsPerStreamMap.put(query.from.objectName, requiredFieldsForQuery);
+ }
+ }
+
+ // Process non-query commands
+ for(Map.Entry<String, BaseCommand> command: commands.entrySet()) {
+ switch(command.getValue().getCommandType()) {
+ case CreateStream:
+ if(topology == null) {
+ throw new RuntimeException("The query engine cannot run CreateStream commands if topology is null");
+ }
+ CreateStream createStream = (CreateStream) command.getValue();
+ Map streamOptions = new HashMap();
+ streamOptions.putAll(this.optionMap);
+ if(createStream.options != null) streamOptions.putAll(createStream.options);
+ StreamAdapter streamAdapter = StreamAdapter.makeAdapter(createStream.spoutName, streamOptions);
+ Stream cStream = streamAdapter.makeStream(
+ topology,
+ topologyName,
+ createStream.streamName,
+ createStream.objectName,
+ createStream.spoutType
+ );
+ cStream = cStream.parallelismHint(options.parallelismHint);
+ if(createStream.deserializer != null) {
+ BaseDeserializer deserializer = BaseDeserializer.makeDeserializer(createStream.deserializer, streamOptions);
+ Set<String> requiredFieldsForInputStream = requiredFieldsPerStreamMap.get(createStream.streamName);
+ List<String> deFields = new ArrayList<>(requiredFieldsForInputStream);
+ // TODO: Is it better to handle metadata fields in a special way?
+ // For example, track them separately, and handle them specially as needed
+ for(String field: cStream.getOutputFields()) {
+ if(deFields.contains(field)) {
+ deFields.remove(field);
+ }
+ }
+ cStream = deserializer.deserialize(cStream, new Fields(deFields));
+ cStream = cStream.project(new Fields(new ArrayList<>(requiredFieldsForInputStream)));
+ }
+ registerInputStream(createStream.streamName, cStream);
+ case Query:
+ // We don't process queries at this point
+ break;
+ case Set:
+ SetCommand setCommand = (SetCommand) command.getValue();
+ optionMap.put(setCommand.key, setCommand.value);
+ break;
+ }
+ }
+
+ if(inputStreams.size() == 0) throw new RuntimeException("There are no registered streams. No queries can be run.");
+
+ // Process input streams according to stream type
+ // TODO: This is deprecated. Handling of deserializing of the stream should be handled by the
+ // create stream command or by the code that creates the stream
+ Map<String, Stream> tupleStreams = new HashMap<>();
+ for(Map.Entry<String, StreamContainer> inputStream: inputStreams.entrySet()) {
+ String InputStreamName = inputStream.getValue().streamName;
+ Set<String> requiredFieldForInputStream = requiredFieldsPerStreamMap.get(InputStreamName);
+ switch(inputStream.getValue().inputStreamType) {
+ case BinaryAvro:
+ tupleStreams.put(inputStream.getKey(),
+ getTupleFromAvro(inputStream.getKey(), inputStream.getValue().stream,
+ new Fields(new ArrayList<>(requiredFieldForInputStream))));
+ break;
+ case Tuple:
+ tupleStreams.put(inputStream.getKey(),
+ inputStream.getValue().stream.project(new Fields(new ArrayList<>(requiredFieldForInputStream))));
+ break;
+ default:
+ throw new RuntimeException(inputStream.getValue().inputStreamType +
+ " is not a supported InputStreamType");
+ }
+ }
+
+ // Build the appropriate components for each query
+ // TODO: No query plan is built, no optimization is done (like a parent query)
+ for(Map.Entry<String, BaseCommand> command: commands.entrySet()) {
+ switch(command.getValue().getCommandType()) {
+ case Query:
+ Query query = (Query) command.getValue();
+ Stream queryStream = tupleStreams.get(query.from.objectName);
+ Set<BaseExpression> keys = getKeys(query.select.expressions);
+ if(query.where != null) queryStream = processFilters(queryStream, query.where);
+ queryStream = processTransforms(queryStream, query.select.expressions);
+ queryStream = queryStream.name("_T");
+ if(hasAggregateExpressions(query.select.expressions)) {
+ queryStream = processAggregators(queryStream, keys, query.select.expressions);
+ queryStream = queryStream.name(command.getKey() + "_A");
+ }
+ InsertIntoClause insert = query.insertInto;
+
+ if (insert.stateName != null && !insert.stateName.equals("")) {
+ persistState(queryStream, insert, query.select.expressions);
+ }
+ queryStream = mapOutputFields(queryStream, query.select.expressions, insert.fields);
+ tupleStreams.put(query.insertInto.objectName, queryStream);
+
+ break;
+ }
+ }
+
+ return tupleStreams;
+ }
+
+ private Set<AggregationExpression> getAggregators(List<BaseExpression> expressions) {
+ Set<AggregationExpression> aggregators = new HashSet<>();
+
+ for(BaseExpression expression: expressions) {
+ switch(expression.getExpressionType()) {
+ case Function:
+ if(((FunctionExpression) expression).getFunctionType() == FunctionType.Aggregation)
+ aggregators.add((AggregationExpression) expression);
+ break;
+ }
+ }
+
+ return aggregators;
+ }
+
+ private Set<String> getFields(BaseExpression expression) {
+ Set<String> fields = new HashSet<>();
+
+ switch(expression.getExpressionType()) {
+ case Field:
+ fields.add(((FieldExpression) expression).fieldName);
+ break;
+ case Function:
+ fields.addAll(getFields(((FunctionExpression) expression).getArguments()));
+ break;
+ }
+
+ return fields;
+ }
+
+ private Set<String> getFields(List<BaseExpression> expressions) {
+ Set<String> fields = new HashSet<>();
+
+ for(BaseExpression expression: expressions) {
+ fields.addAll(getFields(expression));
+ }
+
+ return fields;
+ }
+
+ private Set<BaseExpression> getKeys(List<BaseExpression> expressions) {
+ Set<BaseExpression> keys = new HashSet<>();
+
+ for(BaseExpression expression: expressions) {
+ switch(expression.getExpressionType()) {
+ case Constant:
+ keys.add(expression);
+ break;
+ case Field:
+ keys.add(expression);
+ break;
+ case Function:
+ if(((FunctionExpression) expression).getFunctionType() == FunctionType.Transform)
+ keys.add(expression);
+ break;
+ }
+ }
+
+ return keys;
+ }
+
+ private Stream getTupleFromAvro(String schemaName, Stream inputStream, Fields outputFields) {
+ return inputStream
+ .each(new Fields("bytes"), new GetTupleFromAvro(schemaName, outputFields), outputFields)
+ .project(outputFields);
+ }
+
+ private Boolean hasAggregateExpressions(List<BaseExpression> expressions) {
+ Boolean retVal = false;
+
+ for(BaseExpression expression: expressions) {
+ if(expression.getExpressionType() == ExpressionType.Function
+ && ((FunctionExpression) expression).getFunctionType() == FunctionType.Aggregation) {
+ retVal = true;
+ }
+ }
+
+ return retVal;
+ }
+
+ private Stream mapOutputFields(Stream queryStream, List<BaseExpression> queryExpressions,
+ List<String> outputFields) {
+ List<String> queryFields = new ArrayList<>();
+ if(queryStream.getOutputFields().contains(StreamAdapter.STREAM_METADATA_FIELD)) {
+ queryFields.add(StreamAdapter.STREAM_METADATA_FIELD);
+ for(BaseExpression expression: queryExpressions) queryFields.add(expression.getOutputFieldName());
+
+ queryStream = queryStream.project(new Fields(queryFields));
+ for(int i = 1; i < queryFields.size(); i++) {
+ if(!queryStream.getOutputFields().contains(outputFields.get(i - 1))) {
+ queryStream = queryStream.each(new Fields(queryFields.get(i)), new MapField(), new Fields(outputFields.get(i - 1)));
+ }
+ }
+ } else {
+ for(BaseExpression expression: queryExpressions) queryFields.add(expression.getOutputFieldName());
+
+ queryStream = queryStream.project(new Fields(queryFields));
+ for(int i = 0; i < queryFields.size(); i++) {
+ if(!queryStream.getOutputFields().contains(outputFields.get(i))) {
+ queryStream = queryStream.each(new Fields(queryFields.get(i)), new MapField(), new Fields(outputFields.get(i)));
+ }
+ }
+ }
+
+ List<String> ofClone = new ArrayList<>();
+ if(queryStream.getOutputFields().contains(StreamAdapter.STREAM_METADATA_FIELD)) ofClone.add(StreamAdapter.STREAM_METADATA_FIELD);
+ ofClone.addAll(outputFields);
+ queryStream = queryStream.project(new Fields(ofClone));
+
+ return queryStream;
+ }
+
+ @SuppressWarnings("unchecked")
+ private void persistState(Stream queryStream, InsertIntoClause insert, List<BaseExpression> expressions) {
+ List<String> keyFieldNames = new ArrayList<>();
+ List<String> insertKeys = new ArrayList<>();
+
+ for(int i = 0; i < expressions.size(); i++) {
+ if(expressions.get(i).getExpressionType() != ExpressionType.Function
+ || ((FunctionExpression) expressions.get(i)).getFunctionType() != FunctionType.Aggregation) {
+ insertKeys.add(insert.fields.get(i));
+ keyFieldNames.add(expressions.get(i).getOutputFieldName());
+ }
+ }
+
+ Fields keyFields = new Fields(keyFieldNames);
+
+ Map stateOptions = new HashMap();
+ stateOptions.putAll(this.optionMap);
+ if(insert.options != null) stateOptions.putAll(insert.options);
+ StateAdapter stateAdapter = StateAdapter.makeAdapter(insert.stateName.toLowerCase(), stateOptions);
+
+ if(hasAggregateExpressions(expressions)) {
+ GroupedStream gStream = queryStream.groupBy(keyFields);
+ for (int i = 0; i < expressions.size(); i++) {
+ if (expressions.get(i).getExpressionType() == ExpressionType.Function
+ && ((FunctionExpression) expressions.get(i)).getFunctionType() == FunctionType.Aggregation) {
+ AggregationExpression aExpression = (AggregationExpression) expressions.get(i);
+ StateFactory aggStateFactory =
+ stateAdapter.makeFactory(
+ insert.objectName,
+ insertKeys,
+ insert.fields.get(i),
+ insert.stateType, StateOperationType.AGGREGATE);
+ aExpression.persistentAggregate(gStream, new Fields(expressions.get(i).getOutputFieldName()),
+ aggStateFactory, new Fields(insert.fields.get(i)));
+ }
+ }
+ } else {
+ StateFactory stateFactory = stateAdapter.makeFactory(insert.objectName, insertKeys, null, insert.stateType, StateOperationType.NONAGGREGATE);
+ stateAdapter.partitionPersist(queryStream, stateFactory, keyFields);
+ }
+ }
+
+ private Stream processAggregators(Stream queryStream, Set<BaseExpression> keys, List<BaseExpression> expressions) {
+ Set<AggregationExpression> aggregators = getAggregators(expressions);
+ List<String> keyFieldNames = new ArrayList<>();
+
+ for(BaseExpression expression: keys) {
+ keyFieldNames.add(expression.getOutputFieldName());
+ }
+
+ Fields keyFields = new Fields(keyFieldNames);
+ ChainedPartitionAggregatorDeclarer pChain = queryStream.groupBy(keyFields).chainedAgg();
+
+ for(AggregationExpression expression: aggregators) {
+ String inputField = ((FieldExpression) expression.getArguments().get(0)).fieldName;
+ pChain = expression.partitionAggregate(pChain, new Fields(inputField),
+ new Fields("p_" + expression.getOutputFieldName()));
+ }
+
+ ChainedFullAggregatorDeclarer fChain = pChain.chainEnd().groupBy(keyFields).chainedAgg();
+
+ for(AggregationExpression expression: aggregators) {
+ fChain = expression.aggregate(fChain, new Fields("p_" + expression.getOutputFieldName()),
+ new Fields(expression.getOutputFieldName()));
+ }
+
+ return fChain.chainEnd();
+ }
+
+ private Stream processFilters(Stream queryStream, BaseExpression expression) {
+ List<BaseExpression> expressions = new ArrayList<>();
+
+ expressions.add(expression);
+ queryStream = processTransforms(queryStream, expressions);
+
+ switch (expression.getExpressionType()) {
+ case Constant:
+ Object constant = ((ConstantExpression) expression).constant;
+ if(constant instanceof Boolean) {
+ return queryStream.each(new Fields(expression.getOutputFieldName()), new ValueFilter());
+ } else {
+ throw new RuntimeException("A constant expression can only be used as the top level expression in a WHERE clause if it is Boolean");
+ }
+ case Field:
+ throw new RuntimeException("A field expression is not valid in a WHERE clause");
+ case Function:
+ return queryStream.each(new Fields(expression.getOutputFieldName()), new ValueFilter());
+ default:
+ throw new RuntimeException(expression.getExpressionType() + " is not a valid expression type");
+ }
+ }
+
+ private Stream processTransforms(Stream queryStream, List<BaseExpression> expressions) {
+ List<BaseExpression> unRolledExpressions = new ArrayList<>();
+
+ for(BaseExpression expression: expressions) {
+ List<BaseExpression> temp = expression.unRoll();
+ if(temp != null) unRolledExpressions.addAll(temp);
+ unRolledExpressions.add(expression);
+ }
+
+ for(BaseExpression expression: unRolledExpressions) {
+ switch (expression.getExpressionType()) {
+ case Constant:
+ ConstantExpression cExp = (ConstantExpression) expression;
+ if(!queryStream.getOutputFields().contains(cExp.getOutputFieldName())) {
+ queryStream = queryStream.each(
+ new Fields(),
+ new AddField(cExp.constant),
+ new Fields(cExp.getOutputFieldName()));
+ }
+ break;
+ case Field:
+ // Do nothing, the field is already in the stream
+ break;
+ case Function:
+ if (((FunctionExpression) expression).getFunctionType() == FunctionType.Transform &&
+ !queryStream.getOutputFields().contains(expression.getOutputFieldName())) {
+ TransformExpression tExp = (TransformExpression) expression;
+ queryStream = tExp.transform(queryStream);
+ break;
+ }
+ }
+ }
+
+ return queryStream;
+ }
+
+ public void registerInputStream(String streamName, Stream stream) {
+ inputStreams.put(streamName, new StreamContainer(InputStreamType.Tuple, stream, streamName));
+ }
+
+ public void registerCommands(Iterable<BaseCommand> commands) {
+ for(BaseCommand command: commands) {
+ // TODO: Maaaybe I need a better way of handling the command name
+ switch(command.getCommandType()) {
+ case CreateStream:
+ CreateStream createStream = (CreateStream) command;
+ registerCommand(createStream.streamName, createStream);
+ break;
+ case Query:
+ Query query = (Query) command;
+ registerCommand(query.insertInto.objectName, query);
+ break;
+ case Set:
+ SetCommand setCommand = (SetCommand) command;
+ registerCommand(setCommand.key, command);
+ break;
+ }
+ }
+ }
+
+ public void registerCommand(String commandName, BaseCommand command) {
+ commands.put(commandName, command);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/Topology.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/Topology.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/Topology.java
new file mode 100644
index 0000000..f508169
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/Topology.java
@@ -0,0 +1,126 @@
+package com.jwplayer.sqe;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import org.apache.storm.starter.util.StormRunner;
+import org.apache.storm.trident.TridentTopology;
+import org.apache.storm.Config;
+import org.apache.storm.StormSubmitter;
+import org.apache.storm.generated.StormTopology;
+
+import com.jwplayer.sqe.language.command.BaseCommand;
+import com.jwplayer.sqe.util.FileHelper;
+import com.jwplayer.sqe.util.YamlConfig;
+
+
+public class Topology {
+ List<?> commandPaths;
+ Map<String, Object> config = new HashMap<>();
+ List<?> configPaths;
+ QueryEngine queryEngine;
+ Boolean runLocally;
+ Integer runtime;
+ Config topologyConfig = new Config();
+ String topologyName;
+
+
+
+ public StormTopology createTopology() {
+ TridentTopology topology = new TridentTopology();
+
+ queryEngine.build(topology);
+
+ return topology.build();
+ }
+
+ public OptionParser getOptionParser() {
+ return new OptionParser() {
+ {
+ accepts("config",
+ "Path(s) to one or more YAML config files that is added to the Storm or SQE options")
+ .withRequiredArg();
+ accepts("commands",
+ "Path(s) to one or more JSON files containing SQE commands. Commands are executed in order.")
+ .withRequiredArg();
+ accepts("local", "If set, runs the topology locally");
+ accepts("name", "The name of the topology").withRequiredArg();
+ accepts("runtime",
+ "If running locally, this determines how long the topology runs (in seconds)")
+ .withRequiredArg().defaultsTo("120");
+ accepts("help").forHelp();
+ }
+ };
+ }
+
+ public void loadCommands() throws IOException, URISyntaxException {
+ if (commandPaths != null) {
+ for (Object commandPath : commandPaths) {
+ URI uri = new URI((String) commandPath);
+ String json = FileHelper.loadFileAsString(uri);
+ queryEngine.registerCommands(BaseCommand.load(json));
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public void loadConfig() throws IOException, URISyntaxException {
+ for(Object configPath: configPaths) {
+ Map<String, Object> tempConfig;
+ if (configPath != null) {
+ tempConfig = YamlConfig.getJWConfigAsMap((String) configPath);
+ } else {
+ tempConfig = new HashMap<>();
+ }
+
+ if (tempConfig.containsKey("Storm")) {
+ topologyConfig.putAll((Map) tempConfig.get("Storm"));
+ }
+ if (tempConfig.containsKey("SQE")) {
+ config.putAll((Map) tempConfig.get("SQE"));
+ }
+ }
+
+
+ }
+
+ public void setCommandLineOptions(OptionSet options) {
+ configPaths = options.valuesOf("config");
+ commandPaths = options.valuesOf("commands");
+ runLocally = options.has("local");
+ runtime = Integer.parseInt((String) options.valueOf("runtime"));
+ topologyName = (String) options.valueOf("name");
+ }
+
+ public void run(String args[]) throws Exception {
+ OptionParser parser = getOptionParser();
+ OptionSet options = parser.parse(args);
+
+ if (options.has("help")) {
+ parser.printHelpOn(System.out);
+ System.exit(0);
+ }
+
+ setCommandLineOptions(options);
+ loadConfig();
+ queryEngine = new QueryEngine(topologyName, config);
+ loadCommands();
+
+ if (runLocally) {
+ StormRunner.runTopologyLocally(createTopology(), topologyName, topologyConfig, runtime);
+ } else {
+ StormSubmitter.submitTopology(topologyName, topologyConfig,createTopology());
+ }
+ }
+
+ public static void main(String args[]) throws Exception {
+ Topology topology = new Topology();
+ topology.run(args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/FromClause.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/FromClause.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/FromClause.java
new file mode 100644
index 0000000..e076e71
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/FromClause.java
@@ -0,0 +1,9 @@
+package com.jwplayer.sqe.language.clause;
+
+public class FromClause {
+ public String objectName;
+
+ public FromClause(String objectName) {
+ this.objectName = objectName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/InsertIntoClause.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/InsertIntoClause.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/InsertIntoClause.java
new file mode 100644
index 0000000..e7c757e
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/InsertIntoClause.java
@@ -0,0 +1,23 @@
+package com.jwplayer.sqe.language.clause;
+
+import org.apache.storm.trident.state.StateType;
+
+import java.util.List;
+import java.util.Map;
+
+
+public class InsertIntoClause {
+ public String objectName;
+ public String stateName;
+ public StateType stateType;
+ public List<String> fields;
+ public Map options;
+
+ public InsertIntoClause(String objectName, String stateName, StateType stateType, List<String> fields, Map options) {
+ this.objectName = objectName;
+ this.stateName = stateName;
+ this.stateType = stateType;
+ this.fields = fields;
+ this.options = options;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/SelectClause.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/SelectClause.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/SelectClause.java
new file mode 100644
index 0000000..fe28ee1
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/clause/SelectClause.java
@@ -0,0 +1,13 @@
+package com.jwplayer.sqe.language.clause;
+
+import com.jwplayer.sqe.language.expression.BaseExpression;
+
+import java.util.List;
+
+public class SelectClause {
+ public List<BaseExpression> expressions;
+
+ public SelectClause(List<BaseExpression> expressions) {
+ this.expressions = expressions;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/BaseCommand.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/BaseCommand.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/BaseCommand.java
new file mode 100644
index 0000000..c29e7b5
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/BaseCommand.java
@@ -0,0 +1,57 @@
+package com.jwplayer.sqe.language.command;
+
+import com.google.gson.*;
+import com.jwplayer.sqe.language.expression.BaseExpression;
+
+import java.lang.reflect.Type;
+import java.util.*;
+
+public abstract class BaseCommand {
+ public abstract CommandType getCommandType();
+
+ public static class BaseCommandTypeAdapter implements JsonDeserializer<BaseCommand> {
+ protected Gson gson =
+ new GsonBuilder()
+ .registerTypeAdapter(BaseExpression.class, new BaseExpression.BaseExpressionTypeAdapter())
+ .create();
+
+ @Override
+ public BaseCommand deserialize(JsonElement jsonElement, Type type, JsonDeserializationContext jsonDeserializationContext) throws JsonParseException {
+ if(jsonElement.isJsonObject()) {
+ Set<Map.Entry<String, JsonElement>> entrySet = jsonElement.getAsJsonObject().entrySet();
+ if (entrySet.size() != 1) {
+ throw new RuntimeException("The following JSON should contain one and only one element:\n" + jsonElement.getAsString());
+ }
+ for (Map.Entry<String, JsonElement> entry : entrySet) {
+ return parseCommand(entry.getKey(), entry.getValue());
+ }
+ }
+
+ throw new JsonParseException("Could not parse json:\n" + jsonElement.getAsString());
+ }
+
+ // TODO: Hello factory method, replace?
+ public BaseCommand parseCommand(String commandName, JsonElement command) {
+ switch(commandName.toLowerCase()) {
+ case "createstream":
+ return gson.fromJson(command, CreateStream.class);
+ case "query":
+ return gson.fromJson(command, Query.class);
+ case "set":
+ return gson.fromJson(command, SetCommand.class);
+ default:
+ throw new RuntimeException(commandName + " is not a valid command");
+ }
+ }
+ }
+
+ public static List<BaseCommand> load(String json) {
+ GsonBuilder gson = new GsonBuilder();
+ gson.registerTypeAdapter(BaseExpression.class, new BaseExpression.BaseExpressionTypeAdapter());
+ gson.registerTypeAdapter(BaseCommand.class, new BaseCommand.BaseCommandTypeAdapter());
+
+ BaseCommand[] commands = gson.create().fromJson(json, BaseCommand[].class);
+
+ return Arrays.asList(commands);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/CommandType.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/CommandType.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/CommandType.java
new file mode 100644
index 0000000..2e898fd
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/CommandType.java
@@ -0,0 +1,7 @@
+package com.jwplayer.sqe.language.command;
+
+public enum CommandType {
+ CreateStream,
+ Query,
+ Set
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/CreateStream.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/CreateStream.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/CreateStream.java
new file mode 100644
index 0000000..dddeb3b
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/CreateStream.java
@@ -0,0 +1,20 @@
+package com.jwplayer.sqe.language.command;
+
+import org.apache.storm.trident.state.StateType;
+
+import java.util.Map;
+
+
+public class CreateStream extends BaseCommand {
+ public String streamName;
+ public String objectName;
+ public String spoutName;
+ public StateType spoutType;
+ public String deserializer;
+ public Map options;
+
+ @Override
+ public CommandType getCommandType() {
+ return CommandType.CreateStream;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/Query.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/Query.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/Query.java
new file mode 100644
index 0000000..78879aa
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/Query.java
@@ -0,0 +1,33 @@
+package com.jwplayer.sqe.language.command;
+
+import com.google.gson.GsonBuilder;
+import com.jwplayer.sqe.language.clause.FromClause;
+import com.jwplayer.sqe.language.clause.InsertIntoClause;
+import com.jwplayer.sqe.language.clause.SelectClause;
+import com.jwplayer.sqe.language.expression.BaseExpression;
+
+public class Query extends BaseCommand {
+ public InsertIntoClause insertInto;
+ public SelectClause select;
+ public FromClause from;
+ public BaseExpression where;
+
+ public Query(InsertIntoClause insertInto, SelectClause select, FromClause from, BaseExpression where) {
+ this.insertInto = insertInto;
+ this.select = select;
+ this.from = from;
+ this.where = where;
+ }
+
+ @Deprecated
+ public static Query[] loadQueriesFromJson(String json) {
+ GsonBuilder gson = new GsonBuilder();
+ gson.registerTypeAdapter(BaseExpression.class, new BaseExpression.BaseExpressionTypeAdapter());
+ return gson.create().fromJson(json, Query[].class);
+ }
+
+ @Override
+ public CommandType getCommandType() {
+ return CommandType.Query;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/SetCommand.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/SetCommand.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/SetCommand.java
new file mode 100644
index 0000000..8aba0ec
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/command/SetCommand.java
@@ -0,0 +1,11 @@
+package com.jwplayer.sqe.language.command;
+
+public class SetCommand extends BaseCommand {
+ public String key;
+ public Object value;
+
+ @Override
+ public CommandType getCommandType() {
+ return CommandType.Set;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/BaseExpression.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/BaseExpression.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/BaseExpression.java
new file mode 100644
index 0000000..ce7ac47
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/BaseExpression.java
@@ -0,0 +1,104 @@
+package com.jwplayer.sqe.language.expression;
+
+import com.google.gson.*;
+
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public abstract class BaseExpression {
+ public BaseExpression() { }
+ public abstract ExpressionType getExpressionType();
+ public abstract String getOutputFieldName();
+ public abstract List<BaseExpression> unRoll();
+
+ public static class BaseExpressionTypeAdapter implements JsonDeserializer<BaseExpression>, JsonSerializer<BaseExpression> {
+ @Override
+ public BaseExpression deserialize(JsonElement jsonElement, Type type, JsonDeserializationContext jsonDeserializationContext) throws JsonParseException {
+ if(jsonElement.isJsonPrimitive()) {
+ JsonPrimitive primitive = jsonElement.getAsJsonPrimitive();
+ if(primitive.isString()) return new FieldExpression(jsonElement.getAsJsonPrimitive().getAsString());
+ else return parseConstant(primitive);
+ } else if(jsonElement.isJsonObject()) {
+ Set<Map.Entry<String, JsonElement>> entrySet = jsonElement.getAsJsonObject().entrySet();
+ if (entrySet.size() != 1) {
+ throw new RuntimeException("The following JSON should contain one and only one element:\n" + jsonElement.getAsString());
+ }
+ for (Map.Entry<String, JsonElement> entry : entrySet) {
+ if (entry.getKey().equals("C")) {
+ if(entry.getValue().isJsonNull()) return new ConstantExpression(null);
+ else return parseConstant(entry.getValue().getAsJsonPrimitive());
+ } else {
+ List<BaseExpression> arguments = deserializeArguments(entry.getValue().getAsJsonArray(),
+ type, jsonDeserializationContext);
+ return FunctionExpression.makeFunction(entry.getKey(), arguments);
+ }
+ }
+ } else if(jsonElement.isJsonNull()) {
+ return new ConstantExpression(null);
+ }
+
+ throw new JsonParseException("Could not parse json:\n" + jsonElement.getAsString());
+ }
+
+ private List<BaseExpression> deserializeArguments(JsonArray array, Type type, JsonDeserializationContext jsonDeserializationContext) throws JsonParseException {
+ List<BaseExpression> arguments = new ArrayList<>();
+ for(JsonElement jsonElement: array) {
+ arguments.add(deserialize(jsonElement, type, jsonDeserializationContext));
+ }
+
+ return arguments;
+ }
+
+ private ConstantExpression parseConstant(JsonPrimitive primitive) {
+ if(primitive.isBoolean()) return new ConstantExpression(primitive.getAsBoolean());
+ else if(primitive.isNumber()) {
+ // This is to prevent numerical constants from being parsed as BigDecimal, instead opting
+ // for something slightly more sane and workable.
+ // FIXME: This doesn't work in cultures where . is the 1000s separator
+ if(primitive.getAsString().contains("."))
+ return new ConstantExpression(primitive.getAsNumber().doubleValue());
+ else {
+ long longValue = primitive.getAsNumber().longValue();
+ if(longValue < Integer.MAX_VALUE)
+ return new ConstantExpression(primitive.getAsNumber().intValue());
+ else
+ return new ConstantExpression(primitive.getAsNumber().longValue());
+ }
+ }
+ else if(primitive.isString()) return new ConstantExpression(primitive.getAsString());
+ else throw new JsonParseException("Invalid JSON primitive:\n" + primitive.getAsString());
+ }
+
+ @Override
+ public JsonElement serialize(BaseExpression expression, Type type, JsonSerializationContext jsonSerializationContext) {
+ JsonObject object = new JsonObject();
+ switch(expression.getExpressionType()) {
+ case Constant:
+ ConstantExpression constant = (ConstantExpression) expression;
+
+ if(constant.constant == null) return JsonNull.INSTANCE;
+ else if(constant.constant instanceof Boolean) return new JsonPrimitive((Boolean) constant.constant);
+ else if(constant.constant instanceof Number) return new JsonPrimitive((Number) constant.constant);
+ else if(constant.constant instanceof String) object.addProperty("C", (String) constant.constant);
+ else throw new RuntimeException(constant.constant.getClass().getSimpleName() + " is not a valid constant type");
+
+ return object;
+ case Field:
+ return new JsonPrimitive(((FieldExpression) expression).fieldName);
+ case Function:
+ FunctionExpression function = (FunctionExpression) expression;
+ JsonArray jsonArray = new JsonArray();
+ for(BaseExpression argument: function.getArguments()) {
+ jsonArray.add(serialize(argument, type, jsonSerializationContext));
+ }
+ object.add(function.getFunctionName(), jsonArray);
+ return object;
+ default:
+ throw new RuntimeException(expression.getExpressionType().toString() + " is not a serializable expression type");
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/ConstantExpression.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/ConstantExpression.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/ConstantExpression.java
new file mode 100644
index 0000000..c734717
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/ConstantExpression.java
@@ -0,0 +1,35 @@
+package com.jwplayer.sqe.language.expression;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ConstantExpression extends BaseExpression {
+ public Object constant;
+
+ public ConstantExpression(Object constant) {
+ this.constant = constant;
+ }
+
+ @Override
+ public boolean equals(Object expression) {
+ return (expression.getClass() == this.getClass()
+ && this.constant.equals(((ConstantExpression) expression).constant));
+ }
+
+ @Override
+ public ExpressionType getExpressionType() {
+ return ExpressionType.Constant;
+ }
+
+ @Override
+ public String getOutputFieldName() {
+ return constant == null ?
+ "_C_NULL" :
+ "_C_" + constant.getClass().toString() + "_" + constant.toString();
+ }
+
+ @Override
+ public List<BaseExpression> unRoll() {
+ return new ArrayList<>();
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/ExpressionType.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/ExpressionType.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/ExpressionType.java
new file mode 100644
index 0000000..d625e09
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/ExpressionType.java
@@ -0,0 +1,7 @@
+package com.jwplayer.sqe.language.expression;
+
+public enum ExpressionType {
+ Constant,
+ Field,
+ Function
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FieldExpression.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FieldExpression.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FieldExpression.java
new file mode 100644
index 0000000..e171256
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FieldExpression.java
@@ -0,0 +1,30 @@
+package com.jwplayer.sqe.language.expression;
+
+import java.util.List;
+
+public class FieldExpression extends BaseExpression {
+ public String fieldName;
+
+ public FieldExpression(String fieldName) { this.fieldName = fieldName; }
+
+ @Override
+ public boolean equals(Object expression) {
+ return (expression.getClass() == this.getClass()
+ && this.fieldName.equals(((FieldExpression) expression).fieldName));
+ }
+
+ @Override
+ public ExpressionType getExpressionType() {
+ return ExpressionType.Field;
+ }
+
+ @Override
+ public String getOutputFieldName() {
+ return fieldName;
+ }
+
+ @Override
+ public List<BaseExpression> unRoll() {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FunctionExpression.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FunctionExpression.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FunctionExpression.java
new file mode 100644
index 0000000..3e5d0c8
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FunctionExpression.java
@@ -0,0 +1,119 @@
+package com.jwplayer.sqe.language.expression;
+
+import java.util.*;
+
+import org.apache.storm.tuple.Fields;
+
+
+public abstract class FunctionExpression extends BaseExpression implements Cloneable {
+ protected static ServiceLoader<FunctionExpression> loader = null;
+ protected List<BaseExpression> arguments;
+
+ public FunctionExpression() {
+ }
+
+ @Override
+ public boolean equals(Object expression) {
+ if (expression.getClass() != getClass())
+ return false;
+ FunctionExpression exp = (FunctionExpression) expression;
+ if (exp.getArguments().size() != getArguments().size())
+ return false;
+ boolean isEqual;
+
+ isEqual = getFunctionName().equals(exp.getFunctionName());
+
+ for (int i = 0; i < getArguments().size(); i++) {
+ isEqual = isEqual && getArguments().get(i).equals(exp.getArguments().get(i));
+ }
+
+ return isEqual;
+ }
+
+ public List<BaseExpression> getArguments() {
+ return arguments;
+ }
+
+ @Override
+ public ExpressionType getExpressionType() {
+ return ExpressionType.Function;
+ }
+
+ public abstract String getFunctionName();
+
+ public abstract FunctionType getFunctionType();
+
+ public Fields getInputFields() {
+ List<String> inputFields = new ArrayList<>();
+
+ for (BaseExpression argument : getArguments())
+ inputFields.add(argument.getOutputFieldName());
+
+ return new Fields(inputFields);
+ }
+
+ @Override
+ public String getOutputFieldName() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getFunctionName()).append("(");
+
+ for (int i = 0; i < getArguments().size(); i++) {
+ if (i > 0)
+ sb.append(",");
+ sb.append(getArguments().get(i).getOutputFieldName());
+ }
+
+ sb.append(")");
+
+ return sb.toString();
+ }
+
+ public Fields getOutputFields() {
+ return new Fields(getOutputFieldName());
+ }
+
+ public void setArguments(List<BaseExpression> arguments) {
+ this.arguments = arguments;
+ }
+
+ public List<BaseExpression> unRoll() {
+ List<BaseExpression> retVal = new ArrayList<>();
+
+ for (int i = 0; i < getArguments().size(); i++) {
+ List<BaseExpression> expressionSet = getArguments().get(i).unRoll();
+
+ if (expressionSet != null) {
+ expressionSet.add(getArguments().get(i));
+ getArguments().set(i, new FieldExpression(getArguments().get(i)
+ .getOutputFieldName()));
+ retVal.addAll(expressionSet);
+ }
+ }
+
+ return retVal;
+ }
+
+ public static FunctionExpression makeFunction(String functionName, List<BaseExpression> arguments) {
+ try {
+ if (loader == null) {
+ loader = ServiceLoader.load(FunctionExpression.class);
+ }
+
+ FunctionExpression retVal;
+
+ for (FunctionExpression expression : loader) {
+ if (expression.getFunctionName().equalsIgnoreCase(functionName)) {
+ retVal = (FunctionExpression) expression.clone();
+ retVal.setArguments(arguments);
+
+ return retVal;
+ }
+ }
+ }
+ catch (CloneNotSupportedException e) {
+ throw new RuntimeException(e);
+ }
+
+ throw new RuntimeException(functionName + " is not a valid function expression");
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FunctionType.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FunctionType.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FunctionType.java
new file mode 100644
index 0000000..331b0c2
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/FunctionType.java
@@ -0,0 +1,6 @@
+package com.jwplayer.sqe.language.expression;
+
+public enum FunctionType {
+ Aggregation,
+ Transform
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/AggregationExpression.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/AggregationExpression.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/AggregationExpression.java
new file mode 100644
index 0000000..d1c1719
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/AggregationExpression.java
@@ -0,0 +1,55 @@
+package com.jwplayer.sqe.language.expression.aggregation;
+
+import com.jwplayer.sqe.language.expression.BaseExpression;
+import com.jwplayer.sqe.language.expression.FieldExpression;
+import com.jwplayer.sqe.language.expression.FunctionExpression;
+import com.jwplayer.sqe.language.expression.FunctionType;
+import org.apache.storm.trident.fluent.ChainedFullAggregatorDeclarer;
+import org.apache.storm.trident.fluent.ChainedPartitionAggregatorDeclarer;
+import org.apache.storm.trident.fluent.GroupedStream;
+import org.apache.storm.trident.state.StateFactory;
+import org.apache.storm.tuple.Fields;
+
+import java.util.ArrayList;
+import java.util.List;
+
+
+public abstract class AggregationExpression extends FunctionExpression {
+ public AggregationExpression() {
+ }
+
+ public abstract ChainedFullAggregatorDeclarer aggregate(ChainedFullAggregatorDeclarer stream,
+ Fields inputFields, Fields outputFields);
+
+ @Override
+ public FunctionType getFunctionType() {
+ return FunctionType.Aggregation;
+ }
+
+ public abstract ChainedPartitionAggregatorDeclarer partitionAggregate(ChainedPartitionAggregatorDeclarer stream,
+ Fields inputFields, Fields outputFields);
+
+ public abstract void persistentAggregate(GroupedStream stream, Fields inputFields, StateFactory factory,
+ Fields functionFields);
+
+ // HACK: We override this for aggregation expressions so we only unroll the first argument so we can keep
+ // constants in the 2nd (or 3rd, etc.) positions available to be pulled from the arguments and not the stream.
+ // This is needed because aggregators instantiate the object they are using to aggregate before access to the
+ // stream. This is particularly problematic with HllCreate, where we need a log2m value to create the HLL
+ // object, but want to have this be an argument of the expression. In general, aggregation expressions should
+ // only take a single argument, except in cases like this where the 2nd argument seemed a natural position
+ // for such a value.
+ @Override
+ public List<BaseExpression> unRoll() {
+ List<BaseExpression> retVal = new ArrayList<>();
+ List<BaseExpression> expressionSet = getArguments().get(0).unRoll();
+
+ if(expressionSet != null) {
+ expressionSet.add(getArguments().get(0));
+ getArguments().set(0, new FieldExpression(getArguments().get(0).getOutputFieldName()));
+ retVal.addAll(expressionSet);
+ }
+
+ return retVal;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/CreateHll.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/CreateHll.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/CreateHll.java
new file mode 100644
index 0000000..14089ef
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/CreateHll.java
@@ -0,0 +1,56 @@
+package com.jwplayer.sqe.language.expression.aggregation;
+
+import com.jwplayer.sqe.language.expression.BaseExpression;
+import com.jwplayer.sqe.language.expression.ConstantExpression;
+import com.jwplayer.sqe.language.expression.ExpressionType;
+import com.jwplayer.sqe.trident.aggregator.HllAggregator;
+import com.jwplayer.sqe.trident.aggregator.HllCombinerAggregator;
+import org.apache.storm.trident.fluent.ChainedFullAggregatorDeclarer;
+import org.apache.storm.trident.fluent.ChainedPartitionAggregatorDeclarer;
+import org.apache.storm.trident.fluent.GroupedStream;
+import org.apache.storm.trident.state.StateFactory;
+import org.apache.storm.tuple.Fields;
+
+
+public class CreateHll extends AggregationExpression {
+ public CreateHll() {
+ }
+
+ protected int getLog2m() {
+ if(getArguments().size() == 2) {
+ BaseExpression expression = getArguments().get(1);
+ if(expression.getExpressionType() == ExpressionType.Constant) {
+ ConstantExpression cExp = (ConstantExpression) expression;
+ if(cExp.constant instanceof Number) {
+ return ((Number) cExp.constant).intValue();
+ } else {
+ throw new RuntimeException(String.format("Optional 2nd argument to %s must be a number", getFunctionName()));
+ }
+ } else {
+ throw new RuntimeException(String.format("Optional 2nd argument to %s must be a constant", getFunctionName()));
+ }
+ } else {
+ return 11; // Default log2m
+ }
+ }
+
+ @Override
+ public ChainedFullAggregatorDeclarer aggregate(ChainedFullAggregatorDeclarer stream, Fields inputFields, Fields outputFields) {
+ return stream.aggregate(inputFields, new HllCombinerAggregator(getLog2m()), outputFields);
+ }
+
+ @Override
+ public ChainedPartitionAggregatorDeclarer partitionAggregate(ChainedPartitionAggregatorDeclarer stream, Fields inputFields, Fields outputFields) {
+ return stream.partitionAggregate(inputFields, new HllAggregator(getLog2m()), outputFields);
+ }
+
+ @Override
+ public void persistentAggregate(GroupedStream stream, Fields inputFields, StateFactory factory, Fields functionFields) {
+ stream.persistentAggregate(factory, inputFields, new HllCombinerAggregator(getLog2m()), functionFields);
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "createhll";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/CreateHllp.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/CreateHllp.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/CreateHllp.java
new file mode 100644
index 0000000..e495af0
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/CreateHllp.java
@@ -0,0 +1,35 @@
+package com.jwplayer.sqe.language.expression.aggregation;
+
+import com.jwplayer.sqe.trident.aggregator.HllpAggregator;
+import com.jwplayer.sqe.trident.aggregator.HllpCombinerAggregator;
+import org.apache.storm.trident.fluent.ChainedFullAggregatorDeclarer;
+import org.apache.storm.trident.fluent.ChainedPartitionAggregatorDeclarer;
+import org.apache.storm.trident.fluent.GroupedStream;
+import org.apache.storm.trident.state.StateFactory;
+import org.apache.storm.tuple.Fields;
+
+
+public class CreateHllp extends CreateHll {
+ public CreateHllp() {
+ }
+
+ @Override
+ public ChainedFullAggregatorDeclarer aggregate(ChainedFullAggregatorDeclarer stream, Fields inputFields, Fields outputFields) {
+ return stream.aggregate(inputFields, new HllpCombinerAggregator(getLog2m()), outputFields);
+ }
+
+ @Override
+ public ChainedPartitionAggregatorDeclarer partitionAggregate(ChainedPartitionAggregatorDeclarer stream, Fields inputFields, Fields outputFields) {
+ return stream.partitionAggregate(inputFields, new HllpAggregator(getLog2m()), outputFields);
+ }
+
+ @Override
+ public void persistentAggregate(GroupedStream stream, Fields inputFields, StateFactory factory, Fields functionFields) {
+ stream.persistentAggregate(factory, inputFields, new HllpCombinerAggregator(getLog2m()), functionFields);
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "createhllp";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/Max.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/Max.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/Max.java
new file mode 100644
index 0000000..d9a867b
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/Max.java
@@ -0,0 +1,36 @@
+package com.jwplayer.sqe.language.expression.aggregation;
+
+import com.jwplayer.sqe.trident.aggregator.GetMaximumAggregator;
+import org.apache.storm.trident.fluent.ChainedFullAggregatorDeclarer;
+import org.apache.storm.trident.fluent.ChainedPartitionAggregatorDeclarer;
+import org.apache.storm.trident.fluent.GroupedStream;
+import org.apache.storm.trident.operation.Aggregator;
+import org.apache.storm.trident.operation.CombinerAggregator;
+import org.apache.storm.trident.state.StateFactory;
+import org.apache.storm.tuple.Fields;
+
+
+public class Max extends AggregationExpression {
+ public Max() {
+ }
+
+ @Override
+ public ChainedFullAggregatorDeclarer aggregate(ChainedFullAggregatorDeclarer stream, Fields inputFields, Fields outputFields) {
+ return stream.aggregate(inputFields, (CombinerAggregator) new GetMaximumAggregator(), outputFields);
+ }
+
+ @Override
+ public ChainedPartitionAggregatorDeclarer partitionAggregate(ChainedPartitionAggregatorDeclarer stream, Fields inputFields, Fields outputFields) {
+ return stream.partitionAggregate(inputFields, (Aggregator) new GetMaximumAggregator(), outputFields);
+ }
+
+ @Override
+ public void persistentAggregate(GroupedStream stream, Fields inputFields, StateFactory factory, Fields functionFields) {
+ stream.persistentAggregate(factory, inputFields, new GetMaximumAggregator(), functionFields);
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "max";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/Sum.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/Sum.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/Sum.java
new file mode 100644
index 0000000..d4f5748
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/aggregation/Sum.java
@@ -0,0 +1,33 @@
+package com.jwplayer.sqe.language.expression.aggregation;
+
+import org.apache.storm.trident.fluent.ChainedFullAggregatorDeclarer;
+import org.apache.storm.trident.fluent.ChainedPartitionAggregatorDeclarer;
+import org.apache.storm.trident.fluent.GroupedStream;
+import org.apache.storm.trident.state.StateFactory;
+import org.apache.storm.tuple.Fields;
+
+
+public class Sum extends AggregationExpression {
+ public Sum() {
+ }
+
+ @Override
+ public ChainedFullAggregatorDeclarer aggregate(ChainedFullAggregatorDeclarer stream, Fields inputFields, Fields outputFields) {
+ return stream.aggregate(inputFields, new org.apache.storm.trident.operation.builtin.Sum(), outputFields);
+ }
+
+ @Override
+ public ChainedPartitionAggregatorDeclarer partitionAggregate(ChainedPartitionAggregatorDeclarer stream, Fields inputFields, Fields outputFields) {
+ return stream.partitionAggregate(inputFields, new org.apache.storm.trident.operation.builtin.Sum(), outputFields);
+ }
+
+ @Override
+ public void persistentAggregate(GroupedStream stream, Fields inputFields, StateFactory factory, Fields functionFields) {
+ stream.persistentAggregate(factory, inputFields, new org.apache.storm.trident.operation.builtin.Sum(), functionFields);
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "sum";
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ArithmeticOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ArithmeticOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ArithmeticOperator.java
new file mode 100644
index 0000000..c11a23f
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ArithmeticOperator.java
@@ -0,0 +1,19 @@
+package com.jwplayer.sqe.language.expression.transform;
+
+import com.jwplayer.sqe.trident.function.ProcessArithmeticOperator;
+import org.apache.storm.trident.Stream;
+
+
+public abstract class ArithmeticOperator extends TransformExpression {
+ private ArithmeticOperatorType operatorType = null;
+
+ public ArithmeticOperator(ArithmeticOperatorType operatorType) {
+ super();
+ this.operatorType = operatorType;
+ }
+
+ @Override
+ public Stream transform(Stream stream) {
+ return stream.each(getInputFields(), new ProcessArithmeticOperator(operatorType), getOutputFields());
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ArithmeticOperatorType.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ArithmeticOperatorType.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ArithmeticOperatorType.java
new file mode 100644
index 0000000..7f358d4
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/ArithmeticOperatorType.java
@@ -0,0 +1,9 @@
+package com.jwplayer.sqe.language.expression.transform;
+
+public enum ArithmeticOperatorType {
+ Addition,
+ Division,
+ Modulus,
+ Multiplication,
+ Subtraction
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/DivisionArithmeticOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/DivisionArithmeticOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/DivisionArithmeticOperator.java
new file mode 100644
index 0000000..1dda56c
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/DivisionArithmeticOperator.java
@@ -0,0 +1,13 @@
+package com.jwplayer.sqe.language.expression.transform;
+
+
+public class DivisionArithmeticOperator extends ArithmeticOperator {
+ public DivisionArithmeticOperator() {
+ super(ArithmeticOperatorType.Division);
+ }
+
+ @Override
+ public String getFunctionName() {
+ return "/";
+ }
+}
[06/11] storm git commit: SQE: Apply Apache license headers
Posted by pt...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/IsIn.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/IsIn.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/IsIn.java
index 4d15504..9e27aa9 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/IsIn.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/IsIn.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import org.apache.storm.tuple.Values;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/MapField.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/MapField.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/MapField.java
index a2fe0b1..6d7ac09 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/MapField.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/MapField.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import org.apache.storm.trident.operation.BaseFunction;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ParseDate.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ParseDate.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ParseDate.java
index e964c6f..2fb8ef7 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ParseDate.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ParseDate.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import org.apache.storm.tuple.Values;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ProcessArithmeticOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ProcessArithmeticOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ProcessArithmeticOperator.java
index c86234e..507db31 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ProcessArithmeticOperator.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ProcessArithmeticOperator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import clojure.lang.Numbers;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ProcessLogicalOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ProcessLogicalOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ProcessLogicalOperator.java
index 395eb65..3f035bf 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ProcessLogicalOperator.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ProcessLogicalOperator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import com.jwplayer.sqe.language.expression.transform.predicate.LogicalOperatorType;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/RoundDate.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/RoundDate.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/RoundDate.java
index 780efb3..18e0c9b 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/RoundDate.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/RoundDate.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import org.apache.storm.trident.operation.BaseFunction;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredOpaqueTridentKafkaSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredOpaqueTridentKafkaSpout.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredOpaqueTridentKafkaSpout.java
index 9f01628..c4f5cac 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredOpaqueTridentKafkaSpout.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredOpaqueTridentKafkaSpout.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.spout.kafka;
import org.apache.storm.kafka.Partition;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTransactionalTridentKafkaSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTransactionalTridentKafkaSpout.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTransactionalTridentKafkaSpout.java
index 71e9cca..a908a5d 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTransactionalTridentKafkaSpout.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTransactionalTridentKafkaSpout.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.spout.kafka;
import org.apache.storm.kafka.Partition;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentCollector.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentCollector.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentCollector.java
index 6ad7174..3667052 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentCollector.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentCollector.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.spout.kafka;
import com.jwplayer.sqe.trident.StreamMetadata;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentKafkaEmitter.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentKafkaEmitter.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentKafkaEmitter.java
index d6c5a7a..ca50321 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentKafkaEmitter.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentKafkaEmitter.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.spout.kafka;
import org.apache.storm.kafka.Partition;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/SqeRawFullScheme.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/SqeRawFullScheme.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/SqeRawFullScheme.java
index 538176c..0614773 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/SqeRawFullScheme.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/SqeRawFullScheme.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.spout.kafka;
import com.jwplayer.sqe.language.stream.StreamAdapter;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/BaseGsonSerializer.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/BaseGsonSerializer.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/BaseGsonSerializer.java
index 0a999f1..959f168 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/BaseGsonSerializer.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/BaseGsonSerializer.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.state;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/GsonOpaqueSerializer.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/GsonOpaqueSerializer.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/GsonOpaqueSerializer.java
index ae8863f..00b8f3e 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/GsonOpaqueSerializer.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/GsonOpaqueSerializer.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.state;
import com.google.gson.Gson;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/GsonTransactionalSerializer.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/GsonTransactionalSerializer.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/GsonTransactionalSerializer.java
index c522529..4fbbcdc 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/GsonTransactionalSerializer.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/GsonTransactionalSerializer.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.state;
import com.google.gson.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/mongodb/MongoDBState.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/mongodb/MongoDBState.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/mongodb/MongoDBState.java
index 3bd4e3c..76992a9 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/mongodb/MongoDBState.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/mongodb/MongoDBState.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.state.mongodb;
import com.clearspring.analytics.hash.MurmurHash;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/mongodb/MongoDBStateOptions.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/mongodb/MongoDBStateOptions.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/mongodb/MongoDBStateOptions.java
index ba6643d..5accde7 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/mongodb/MongoDBStateOptions.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/mongodb/MongoDBStateOptions.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.state.mongodb;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/redis/RedisKeyFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/redis/RedisKeyFactory.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/redis/RedisKeyFactory.java
index 42881a9..88d95d7 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/redis/RedisKeyFactory.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/redis/RedisKeyFactory.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.state.redis;
import org.apache.storm.redis.trident.state.KeyFactory;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/FileHelper.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/FileHelper.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/FileHelper.java
index 6e8d7ec..359f478 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/FileHelper.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/FileHelper.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.util;
import org.apache.commons.io.IOUtils;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/YamlConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/YamlConfig.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/YamlConfig.java
index deaef02..1102373 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/YamlConfig.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/YamlConfig.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.util;
import org.apache.log4j.Logger;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourceFileSystem.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourceFileSystem.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourceFileSystem.java
index 572ccd1..ca9239d 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourceFileSystem.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourceFileSystem.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.util.file.resource;
import com.google.common.collect.ImmutableList;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourceFileSystemProvider.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourceFileSystemProvider.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourceFileSystemProvider.java
index b84f035..2a65606 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourceFileSystemProvider.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourceFileSystemProvider.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.util.file.resource;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourcePath.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourcePath.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourcePath.java
index 0d5960a..16392a1 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourcePath.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/file/resource/ResourcePath.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.util.file.resource;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/IntegrationTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/IntegrationTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/IntegrationTest.java
index 768a89f..d670088 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/IntegrationTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/IntegrationTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/KafkaReplaysIntegrationTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/KafkaReplaysIntegrationTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/KafkaReplaysIntegrationTest.java
index 9942ace..ca57cd0 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/KafkaReplaysIntegrationTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/KafkaReplaysIntegrationTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe;
import com.jwplayer.sqe.trident.StreamMetadata;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/command/CommandDeserializationTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/command/CommandDeserializationTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/command/CommandDeserializationTest.java
index 8b5ee25..ddfbd79 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/command/CommandDeserializationTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/command/CommandDeserializationTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.command;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/expression/ExpressionDeserializationTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/expression/ExpressionDeserializationTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/expression/ExpressionDeserializationTest.java
index b483443..7f67b2e 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/expression/ExpressionDeserializationTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/expression/ExpressionDeserializationTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression;
import com.google.gson.GsonBuilder;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/expression/FunctionExpressionTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/expression/FunctionExpressionTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/expression/FunctionExpressionTest.java
index 52ee8ef..9c2cbe9 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/expression/FunctionExpressionTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/language/expression/FunctionExpressionTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/GetTuplesFilter.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/GetTuplesFilter.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/GetTuplesFilter.java
index 46361ae..906d6bd 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/GetTuplesFilter.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/GetTuplesFilter.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident;
import org.apache.storm.trident.operation.BaseFilter;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/GetMaximumAggregatorTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/GetMaximumAggregatorTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/GetMaximumAggregatorTest.java
index 5dc7eb6..8f88adb 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/GetMaximumAggregatorTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/GetMaximumAggregatorTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.aggregator;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllAggregatorTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllAggregatorTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllAggregatorTest.java
index 7d8e826..fc74b98 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllAggregatorTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllAggregatorTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.aggregator;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllCombinerAggregatorTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllCombinerAggregatorTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllCombinerAggregatorTest.java
index c616a7a..2ba5663 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllCombinerAggregatorTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllCombinerAggregatorTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.aggregator;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllpAggregatorTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllpAggregatorTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllpAggregatorTest.java
index 03705b2..378fc57 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllpAggregatorTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllpAggregatorTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.aggregator;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllpCombinerAggregatorTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllpCombinerAggregatorTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllpCombinerAggregatorTest.java
index 5e59af5..daa42ee 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllpCombinerAggregatorTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/aggregator/HllpCombinerAggregatorTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.aggregator;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/AddFieldTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/AddFieldTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/AddFieldTest.java
index 0767e55..ee7dad9 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/AddFieldTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/AddFieldTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/CompareNumbersTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/CompareNumbersTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/CompareNumbersTest.java
index 137689d..3cdd2ea 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/CompareNumbersTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/CompareNumbersTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/CompareObjectsTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/CompareObjectsTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/CompareObjectsTest.java
index 07c84ae..89b8853 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/CompareObjectsTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/CompareObjectsTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/EvaluateRegularExpressionTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/EvaluateRegularExpressionTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/EvaluateRegularExpressionTest.java
index e081bce..69650ad 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/EvaluateRegularExpressionTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/EvaluateRegularExpressionTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ExpandKeysTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ExpandKeysTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ExpandKeysTest.java
index 3688b78..1cd4f9e 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ExpandKeysTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ExpandKeysTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import com.jwplayer.sqe.trident.ListValuesCollector;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ExpandValuesTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ExpandValuesTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ExpandValuesTest.java
index 45e374c..5604152 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ExpandValuesTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ExpandValuesTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import com.jwplayer.sqe.trident.ListValuesCollector;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/FormatDateTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/FormatDateTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/FormatDateTest.java
index 776a9c2..e12ea4d 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/FormatDateTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/FormatDateTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetCardinalityEstimationTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetCardinalityEstimationTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetCardinalityEstimationTest.java
index 73ab2d3..7984fdb 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetCardinalityEstimationTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetCardinalityEstimationTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import com.clearspring.analytics.stream.cardinality.HyperLogLog;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetIfTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetIfTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetIfTest.java
index 730a8b8..8a408d8 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetIfTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetIfTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import com.jwplayer.sqe.trident.SingleValuesCollector;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetMapValueTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetMapValueTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetMapValueTest.java
index e5420e4..bef57ac 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetMapValueTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetMapValueTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import static org.junit.Assert.assertEquals;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetTimeTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetTimeTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetTimeTest.java
index 57dc090..007a20c 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetTimeTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/GetTimeTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import com.jwplayer.sqe.trident.SingleValuesCollector;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/HashTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/HashTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/HashTest.java
index 98ff587..3c84d80 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/HashTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/HashTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import com.google.common.primitives.Longs;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/IsInTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/IsInTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/IsInTest.java
index aecb919..0df18ea 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/IsInTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/IsInTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import com.jwplayer.sqe.trident.SingleValuesCollector;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ParseDateTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ParseDateTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ParseDateTest.java
index 9f8b351..36991ce 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ParseDateTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ParseDateTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ProcessArithmeticOperatorTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ProcessArithmeticOperatorTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ProcessArithmeticOperatorTest.java
index c480dea..f432e76 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ProcessArithmeticOperatorTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ProcessArithmeticOperatorTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ProcessLogicalOperatorTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ProcessLogicalOperatorTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ProcessLogicalOperatorTest.java
index bfd641d..2411bac 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ProcessLogicalOperatorTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/ProcessLogicalOperatorTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/RoundDateTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/RoundDateTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/RoundDateTest.java
index 2914b89..e7bcd10 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/RoundDateTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/function/RoundDateTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentCollectorTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentCollectorTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentCollectorTest.java
index ff9b466..2218339 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentCollectorTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentCollectorTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.spout.kafka;
import static org.junit.Assert.*;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/spout/kafka/SqeRawFullSchemeTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/spout/kafka/SqeRawFullSchemeTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/spout/kafka/SqeRawFullSchemeTest.java
index 0ddb1f3..a67d02b 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/spout/kafka/SqeRawFullSchemeTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/trident/spout/kafka/SqeRawFullSchemeTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.spout.kafka;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/FileHelperTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/FileHelperTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/FileHelperTest.java
index bca7ec8..bb88214 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/FileHelperTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/FileHelperTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.util;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/KafkaTestServer.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/KafkaTestServer.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/KafkaTestServer.java
index e7bd2ac..e065cc9 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/KafkaTestServer.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/KafkaTestServer.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.util;
import kafka.admin.AdminUtils;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/YamlConfigTest.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/YamlConfigTest.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/YamlConfigTest.java
index 2da8086..6dec769 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/YamlConfigTest.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/YamlConfigTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.util;
import org.junit.Before;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/ZookeeperTestServer.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/ZookeeperTestServer.java b/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/ZookeeperTestServer.java
index b532686..8d6e30e 100644
--- a/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/ZookeeperTestServer.java
+++ b/external/storm-sqe/src/test/java/com/jwplayer/sqe/util/ZookeeperTestServer.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.util;
import kafka.utils.ZkUtils;
[03/11] storm git commit: initial SQE import
Posted by pt...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/GetMaximumAggregator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/GetMaximumAggregator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/GetMaximumAggregator.java
new file mode 100644
index 0000000..7505b93
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/GetMaximumAggregator.java
@@ -0,0 +1,73 @@
+package com.jwplayer.sqe.trident.aggregator;
+
+import clojure.lang.Numbers;
+import org.apache.storm.trident.operation.BaseAggregator;
+import org.apache.storm.trident.operation.CombinerAggregator;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Values;
+
+
+public class GetMaximumAggregator extends BaseAggregator<GetMaximumAggregator.MaximumState> implements CombinerAggregator<Comparable> {
+ public static class MaximumState {
+ Comparable comparable;
+
+ public MaximumState(Comparable comparable) {
+ this.comparable = comparable;
+ }
+ }
+
+ @Override
+ public GetMaximumAggregator.MaximumState init(Object txID, TridentCollector collector) {
+ return new MaximumState(null);
+ }
+
+ @Override
+ public void aggregate(GetMaximumAggregator.MaximumState state, TridentTuple tuple, TridentCollector collector) {
+ if(tuple.get(0) instanceof Comparable) {
+ state.comparable = combine(state.comparable, (Comparable) tuple.get(0));
+ } else {
+ throw new RuntimeException(
+ String.format("All values sent to the Max function must implement Comparable<T>. The given value's type is %s.",
+ tuple.get(0).getClass().toString()));
+ }
+ }
+
+ @Override
+ public void complete(GetMaximumAggregator.MaximumState state, TridentCollector collector) {
+ collector.emit(new Values(state.comparable));
+ }
+
+ @Override
+ public Comparable init(TridentTuple tuple) {
+ if(tuple.get(0) instanceof Comparable) {
+ return (Comparable) tuple.get(0);
+ } else {
+ throw new RuntimeException(
+ String.format("All values sent to the Max function must implement Comparable<T>. The given value's type is %s.",
+ tuple.get(0).getClass().toString()));
+ }
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Comparable combine(Comparable comp1, Comparable comp2) {
+ if(comp1 == null) {
+ return comp2;
+ } else if(comp2 == null) {
+ return comp1;
+ } else if(comp1 instanceof Number && comp2 instanceof Number) {
+ // Handle Numbers specially so we can compare cross type
+ return (Comparable) Numbers.max(comp1, comp2);
+ } else if(comp1.compareTo(comp2) < 0) {
+ return comp2;
+ } else {
+ return comp1;
+ }
+ }
+
+ @Override
+ public Comparable zero() {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllAggregator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllAggregator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllAggregator.java
new file mode 100644
index 0000000..0b62dfc
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllAggregator.java
@@ -0,0 +1,17 @@
+package com.jwplayer.sqe.trident.aggregator;
+
+import com.clearspring.analytics.stream.cardinality.HyperLogLog;
+import org.apache.storm.trident.operation.TridentCollector;
+
+public class HllAggregator extends CardinalityEstimatorAggregator {
+ private int log2m;
+
+ public HllAggregator(int log2m) {
+ this.log2m = log2m;
+ }
+
+ @Override
+ public HyperLogLog init(Object o, TridentCollector collector) {
+ return new HyperLogLog(log2m);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllCombinerAggregator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllCombinerAggregator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllCombinerAggregator.java
new file mode 100644
index 0000000..ae0ca59
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllCombinerAggregator.java
@@ -0,0 +1,36 @@
+package com.jwplayer.sqe.trident.aggregator;
+
+import com.clearspring.analytics.stream.cardinality.CardinalityMergeException;
+import com.clearspring.analytics.stream.cardinality.HyperLogLog;
+
+import java.io.IOException;
+
+
+public class HllCombinerAggregator extends CardinalityEstimatorCombinerAggregator {
+ private int log2m;
+
+ public HllCombinerAggregator(int log2m) {
+ this.log2m = log2m;
+ }
+
+ @Override
+ public byte[] combine(byte[] bitmap1, byte[] bitmap2) {
+ try {
+ HyperLogLog hll1 = HyperLogLog.Builder.build(bitmap1);
+ HyperLogLog hll2 = HyperLogLog.Builder.build(bitmap2);
+ hll1.addAll(hll2);
+ return hll1.getBytes();
+ } catch (CardinalityMergeException |IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @Override
+ public byte[] zero() {
+ try {
+ return (new HyperLogLog(log2m)).getBytes();
+ } catch(IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllpAggregator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllpAggregator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllpAggregator.java
new file mode 100644
index 0000000..5fbea2f
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllpAggregator.java
@@ -0,0 +1,18 @@
+package com.jwplayer.sqe.trident.aggregator;
+
+import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
+import org.apache.storm.trident.operation.TridentCollector;
+
+
+public class HllpAggregator extends CardinalityEstimatorAggregator {
+ private int log2m;
+
+ public HllpAggregator(int log2m) {
+ this.log2m = log2m;
+ }
+
+ @Override
+ public HyperLogLogPlus init(Object o, TridentCollector collector) {
+ return new HyperLogLogPlus(log2m, log2m);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllpCombinerAggregator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllpCombinerAggregator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllpCombinerAggregator.java
new file mode 100644
index 0000000..77078c6
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllpCombinerAggregator.java
@@ -0,0 +1,35 @@
+package com.jwplayer.sqe.trident.aggregator;
+
+import com.clearspring.analytics.stream.cardinality.CardinalityMergeException;
+import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
+
+import java.io.IOException;
+
+public class HllpCombinerAggregator extends CardinalityEstimatorCombinerAggregator {
+ private int log2m;
+
+ public HllpCombinerAggregator(int log2m) {
+ this.log2m = log2m;
+ }
+
+ @Override
+ public byte[] combine(byte[] bitmap1, byte[] bitmap2) {
+ try {
+ HyperLogLogPlus hllp1 = HyperLogLogPlus.Builder.build(bitmap1);
+ HyperLogLogPlus hllp2 = HyperLogLogPlus.Builder.build(bitmap2);
+ hllp1.addAll(hllp2);
+ return hllp1.getBytes();
+ } catch (CardinalityMergeException |IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @Override
+ public byte[] zero() {
+ try {
+ return (new HyperLogLogPlus(log2m, log2m)).getBytes();
+ } catch(IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/AddField.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/AddField.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/AddField.java
new file mode 100644
index 0000000..fb598a2
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/AddField.java
@@ -0,0 +1,19 @@
+package com.jwplayer.sqe.trident.function;
+
+import org.apache.storm.tuple.Values;
+import org.apache.storm.trident.operation.BaseFunction;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+
+public class AddField extends BaseFunction {
+ private Object value;
+
+ public AddField(Object value) {
+ this.value = value;
+ }
+
+ @Override
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ collector.emit(new Values(value));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/CompareNumbers.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/CompareNumbers.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/CompareNumbers.java
new file mode 100644
index 0000000..8ddb99a
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/CompareNumbers.java
@@ -0,0 +1,45 @@
+package com.jwplayer.sqe.trident.function;
+
+import clojure.lang.Numbers;
+import com.jwplayer.sqe.language.expression.transform.predicate.NumberComparisonType;
+import org.apache.storm.trident.operation.BaseFunction;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Values;
+
+
+public class CompareNumbers extends BaseFunction {
+ private NumberComparisonType comparisonType = null;
+
+ public CompareNumbers(NumberComparisonType comparisonType) {
+ this.comparisonType = comparisonType;
+ }
+
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ Number value1 = (Number) tuple.get(0);
+ Number value2 = (Number) tuple.get(1);
+
+ switch (comparisonType) {
+ case Equal:
+ collector.emit(new Values(Numbers.equiv(value1, value2)));
+ break;
+ case GreaterThan:
+ collector.emit(new Values(Numbers.gt(value1, value2)));
+ break;
+ case GreaterThanOrEqual:
+ collector.emit(new Values(Numbers.gte(value1, value2)));
+ break;
+ case LessThan:
+ collector.emit(new Values(Numbers.lt(value1, value2)));
+ break;
+ case LessThanOrEqual:
+ collector.emit(new Values(Numbers.lte(value1, value2)));
+ break;
+ case NotEqual:
+ collector.emit(new Values(!Numbers.equiv(value1, value2)));
+ break;
+ default:
+ throw new UnsupportedOperationException(comparisonType.toString() + " is not supported");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/CompareObjects.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/CompareObjects.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/CompareObjects.java
new file mode 100644
index 0000000..5625c7c
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/CompareObjects.java
@@ -0,0 +1,25 @@
+package com.jwplayer.sqe.trident.function;
+
+import com.jwplayer.sqe.language.expression.transform.predicate.NumberComparisonType;
+import org.apache.storm.trident.operation.BaseFunction;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Values;
+
+
+public class CompareObjects extends BaseFunction {
+ private CompareNumbers numberComparator = new CompareNumbers(NumberComparisonType.Equal);
+
+ @Override
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ Object value1 = tuple.get(0);
+ Object value2 = tuple.get(1);
+
+ if (value1 instanceof Number && value2 instanceof Number) {
+ numberComparator.execute(tuple, collector);
+ } else {
+ if(value1 == null) collector.emit(new Values(value2 == null));
+ else collector.emit(new Values(value1.equals(value2)));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ConvertMetadataToBytes.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ConvertMetadataToBytes.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ConvertMetadataToBytes.java
new file mode 100644
index 0000000..e5eb525
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ConvertMetadataToBytes.java
@@ -0,0 +1,17 @@
+package com.jwplayer.sqe.trident.function;
+
+import com.jwplayer.sqe.trident.StreamMetadata;
+import org.apache.storm.trident.operation.BaseFunction;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Values;
+
+
+public class ConvertMetadataToBytes extends BaseFunction {
+ @Override
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ StreamMetadata metadata = (StreamMetadata) tuple.get(0);
+
+ collector.emit(new Values(metadata.toBytes()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/EvaluateRegularExpression.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/EvaluateRegularExpression.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/EvaluateRegularExpression.java
new file mode 100644
index 0000000..739879e
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/EvaluateRegularExpression.java
@@ -0,0 +1,30 @@
+package com.jwplayer.sqe.trident.function;
+
+import org.apache.storm.trident.operation.BaseFunction;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Values;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+
+public class EvaluateRegularExpression extends BaseFunction {
+ Map<String, Pattern> patternCache = new HashMap<>();
+
+ @Override
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ Pattern pattern;
+ String regex = tuple.getString(1);
+
+ if(patternCache.containsKey(regex)) {
+ pattern = patternCache.get(regex);
+ } else {
+ pattern = Pattern.compile(regex);
+ patternCache.put(regex, pattern);
+ }
+
+ collector.emit(new Values(pattern.matcher(tuple.getString(0)).matches()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ExpandKeys.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ExpandKeys.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ExpandKeys.java
new file mode 100644
index 0000000..4178679
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ExpandKeys.java
@@ -0,0 +1,29 @@
+package com.jwplayer.sqe.trident.function;
+
+import org.apache.avro.util.Utf8;
+import org.apache.storm.trident.operation.BaseFunction;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Values;
+
+import java.util.Map;
+import java.util.TreeSet;
+
+
+public class ExpandKeys extends BaseFunction {
+ @Override
+ @SuppressWarnings("unchecked")
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ Map map = (Map) tuple.get(0);
+ TreeSet set = new TreeSet(map.keySet());
+
+ for(Object key: set) {
+ // Avro strings are stored using a special Avro type instead of using Java primitives
+ if(key instanceof Utf8) {
+ collector.emit(new Values(key.toString()));
+ } else {
+ collector.emit(new Values(key));
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ExpandValues.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ExpandValues.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ExpandValues.java
new file mode 100644
index 0000000..4a26d4c
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ExpandValues.java
@@ -0,0 +1,30 @@
+package com.jwplayer.sqe.trident.function;
+
+import org.apache.avro.util.Utf8;
+import org.apache.storm.trident.operation.BaseFunction;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Values;
+
+import java.util.Map;
+import java.util.TreeSet;
+
+
+public class ExpandValues extends BaseFunction {
+ @Override
+ @SuppressWarnings("unchecked")
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ Map map = (Map) tuple.get(0);
+ TreeSet set = new TreeSet(map.keySet());
+
+ for(Object key: set) {
+ // Avro strings are stored using a special Avro type instead of using Java primitives
+ Object value = map.get(key);
+ if(value instanceof Utf8) {
+ collector.emit(new Values(value.toString()));
+ } else {
+ collector.emit(new Values(value));
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/FormatDate.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/FormatDate.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/FormatDate.java
new file mode 100644
index 0000000..6b3762b
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/FormatDate.java
@@ -0,0 +1,26 @@
+package com.jwplayer.sqe.trident.function;
+
+import org.apache.storm.trident.operation.BaseFunction;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Values;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.TimeZone;
+
+
+public class FormatDate extends BaseFunction {
+ @Override
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ try {
+ SimpleDateFormat dateFormat = new SimpleDateFormat(tuple.getString(1));
+ dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ String date = dateFormat.format((Date) tuple.get(0));
+
+ collector.emit(new Values(date));
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetCardinalityEstimation.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetCardinalityEstimation.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetCardinalityEstimation.java
new file mode 100644
index 0000000..9a5a41f
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetCardinalityEstimation.java
@@ -0,0 +1,33 @@
+package com.jwplayer.sqe.trident.function;
+
+import com.clearspring.analytics.stream.cardinality.HyperLogLog;
+import org.apache.storm.trident.operation.BaseFunction;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Values;
+
+import java.io.IOException;
+
+
+public class GetCardinalityEstimation extends BaseFunction {
+ @Override
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ Object object = tuple.get(0);
+ HyperLogLog hll;
+
+ if(object instanceof HyperLogLog) {
+ hll = (HyperLogLog) object;
+ }
+ else if(object instanceof byte[]) {
+ try {
+ hll = HyperLogLog.Builder.build((byte[]) object);
+ } catch(IOException ex) {
+ throw new RuntimeException("Could not build HyperLogLog object from byte array", ex);
+ }
+ } else {
+ throw new RuntimeException(object.getClass() + " is not an appropriate HLL object or byte array");
+ }
+
+ collector.emit(new Values(hll.cardinality()));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetIf.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetIf.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetIf.java
new file mode 100644
index 0000000..a41d4cf
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetIf.java
@@ -0,0 +1,17 @@
+package com.jwplayer.sqe.trident.function;
+
+import org.apache.storm.tuple.Values;
+import org.apache.storm.trident.operation.BaseFunction;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+
+public class GetIf extends BaseFunction {
+ @Override
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ Boolean predicateValue = (Boolean) tuple.get(0);
+ Object trueValue = tuple.get(1);
+ Object falseValue = tuple.get(2);
+
+ collector.emit(new Values(predicateValue ? trueValue : falseValue));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetMapValue.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetMapValue.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetMapValue.java
new file mode 100644
index 0000000..21e7eba
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetMapValue.java
@@ -0,0 +1,24 @@
+package com.jwplayer.sqe.trident.function;
+
+import java.util.Map;
+import org.apache.storm.trident.operation.BaseFunction;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Values;
+
+
+public class GetMapValue extends BaseFunction{
+
+ @Override
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ // TODO Auto-generated method stub
+ @SuppressWarnings("unchecked")
+ Map<Object, Object> m = (Map<Object, Object>) tuple.get(0);
+ Object key = tuple.get(1);
+
+ if (m.containsKey(key)){
+ collector.emit(new Values(m.get(key)));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetTime.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetTime.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetTime.java
new file mode 100644
index 0000000..ddbb0a3
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetTime.java
@@ -0,0 +1,35 @@
+package com.jwplayer.sqe.trident.function;
+
+import org.apache.log4j.Logger;
+import org.apache.storm.trident.operation.BaseFunction;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Values;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.TimeZone;
+
+
+public class GetTime extends BaseFunction {
+ private static final Logger LOG = Logger.getLogger(GetTime.class);
+
+ @Override
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ try {
+ Date isoDate;
+
+ if(tuple.size() == 2) {
+ SimpleDateFormat isoDateFormat = new SimpleDateFormat(tuple.getString(1));
+ isoDateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ isoDate = isoDateFormat.parse(tuple.getString(0));
+ } else {
+ isoDate = (Date) tuple.get(0);
+ }
+
+ collector.emit(new Values(isoDate.getTime()));
+ } catch (Exception ex) {
+ LOG.error(ex.getMessage(), ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetTupleFromAvro.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetTupleFromAvro.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetTupleFromAvro.java
new file mode 100644
index 0000000..94544e9
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetTupleFromAvro.java
@@ -0,0 +1,70 @@
+package com.jwplayer.sqe.trident.function;
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.io.DatumReader;
+import org.apache.avro.io.Decoder;
+import org.apache.avro.io.DecoderFactory;
+import org.apache.avro.reflect.ReflectData;
+import org.apache.avro.util.Utf8;
+import org.apache.storm.trident.operation.BaseFunction;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Values;
+
+import java.io.ByteArrayInputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+
+public class GetTupleFromAvro extends BaseFunction {
+ private Fields fields;
+ private Schema schema = null;
+ private String schemaName;
+
+ public GetTupleFromAvro(String schemaName, Fields fields) {
+ this.fields = fields;
+ this.schemaName = schemaName;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ Values values = new Values();
+
+ try {
+ if(schema == null) schema = ReflectData.get().getSchema(Class.forName(schemaName));
+ DatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
+ ByteArrayInputStream in = new ByteArrayInputStream(tuple.getBinary(1));
+ Decoder decoder = DecoderFactory.get().directBinaryDecoder(in, null);
+ GenericRecord record = reader.read(null, decoder);
+
+ for(String field : fields) {
+ Object value = record.get(field);
+ // Avro strings are stored using a special Avro type instead of using Java primitives
+ if(value instanceof Utf8) {
+ values.add(value.toString());
+ } else if(value instanceof Map<?, ?>) {
+ // Due to type erasure, generic type parameter can't be generalized for whole map
+ Map<Object, Object> map = new HashMap<>();
+ for (Map.Entry<Object, Object> entry : ((Map<Object, Object>)value).entrySet()) {
+ Object key = entry.getKey();
+ Object newKey = key instanceof Utf8 ? key.toString() : key;
+ Object val = entry.getValue();
+ Object newVal = val instanceof Utf8 ? val.toString() : val;
+ map.put(newKey, newVal);
+ }
+ values.add(map);
+ } else {
+ values.add(value);
+ }
+ }
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+
+ collector.emit(values);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/Hash.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/Hash.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/Hash.java
new file mode 100644
index 0000000..dfbd85e
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/Hash.java
@@ -0,0 +1,37 @@
+package com.jwplayer.sqe.trident.function;
+
+import com.clearspring.analytics.hash.MurmurHash;
+import com.google.common.primitives.Longs;
+import org.apache.storm.trident.operation.BaseFunction;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Values;
+
+
+public class Hash extends BaseFunction {
+
+ @Override
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ String hashName = "murmur2";
+ Object object;
+
+ if(tuple.size() == 1) {
+ object = tuple.get(0);
+ } else {
+ hashName = tuple.getString(0);
+ object = tuple.get(1);
+ }
+
+ // Emits a hash as byte[]
+ switch(hashName.toLowerCase()) {
+ // Uses Clearsprings Murmur 2.0 implementation
+ case "murmur2":
+ Long m2Hash = MurmurHash.hash64(object);
+ byte[] m2Bytes = Longs.toByteArray(m2Hash);
+ collector.emit(new Values(m2Bytes));
+ break;
+ default:
+ throw new UnsupportedOperationException(hashName + "is not a supported hashing algorithm");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/IsIn.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/IsIn.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/IsIn.java
new file mode 100644
index 0000000..4d15504
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/IsIn.java
@@ -0,0 +1,31 @@
+package com.jwplayer.sqe.trident.function;
+
+import org.apache.storm.tuple.Values;
+import clojure.lang.Numbers;
+import org.apache.storm.trident.operation.BaseFunction;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+
+public class IsIn extends BaseFunction {
+ @Override
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ if(tuple.size() < 2) throw new RuntimeException("IfIn function expects more than 1 value on the tuple");
+
+ Object value = tuple.get(0);
+ Boolean valueFound = false;
+
+ if(value instanceof Number) {
+ Number number = (Number) value;
+
+ for(int i = 1; i < tuple.size(); i++) {
+ if(Numbers.equiv(number, (Number) tuple.get(i))) valueFound = true;
+ }
+ } else {
+ for(int i = 1; i < tuple.size(); i++) {
+ if(value.equals(tuple.get(i))) valueFound = true;
+ }
+ }
+
+ collector.emit(new Values(valueFound));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/MapField.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/MapField.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/MapField.java
new file mode 100644
index 0000000..a2fe0b1
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/MapField.java
@@ -0,0 +1,14 @@
+package com.jwplayer.sqe.trident.function;
+
+import org.apache.storm.trident.operation.BaseFunction;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Values;
+
+
+public class MapField extends BaseFunction {
+ @Override
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ collector.emit(new Values(tuple.get(0)));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ParseDate.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ParseDate.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ParseDate.java
new file mode 100644
index 0000000..e964c6f
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ParseDate.java
@@ -0,0 +1,26 @@
+package com.jwplayer.sqe.trident.function;
+
+import org.apache.storm.tuple.Values;
+import org.apache.storm.trident.operation.BaseFunction;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.TimeZone;
+
+
+public class ParseDate extends BaseFunction {
+ @Override
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ try {
+ SimpleDateFormat dateFormat = new SimpleDateFormat(tuple.getString(1));
+ dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ Date date = dateFormat.parse(tuple.getString(0));
+
+ collector.emit(new Values(date));
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ProcessArithmeticOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ProcessArithmeticOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ProcessArithmeticOperator.java
new file mode 100644
index 0000000..c86234e
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ProcessArithmeticOperator.java
@@ -0,0 +1,46 @@
+package com.jwplayer.sqe.trident.function;
+
+import clojure.lang.Numbers;
+import clojure.lang.Ratio;
+import com.jwplayer.sqe.language.expression.transform.ArithmeticOperatorType;
+import org.apache.storm.trident.operation.BaseFunction;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Values;
+
+
+public class ProcessArithmeticOperator extends BaseFunction {
+ private ArithmeticOperatorType operatorType = null;
+
+ public ProcessArithmeticOperator(ArithmeticOperatorType operatorType) {
+ this.operatorType = operatorType;
+ }
+
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ Number value1 = (Number) tuple.get(0);
+ Number value2 = (Number) tuple.get(1);
+
+ switch(operatorType) {
+ case Addition:
+ collector.emit(new Values(Numbers.add(value1, value2)));
+ break;
+ case Division:
+ Number retVal = Numbers.divide(value1, value2);
+
+ if(retVal instanceof Ratio) collector.emit(new Values(retVal.doubleValue()));
+ else collector.emit(new Values(retVal));
+ break;
+ case Modulus:
+ collector.emit(new Values(Numbers.remainder(value1, value2)));
+ break;
+ case Multiplication:
+ collector.emit(new Values(Numbers.multiply(value1, value2)));
+ break;
+ case Subtraction:
+ collector.emit(new Values(Numbers.minus(value1, value2)));
+ break;
+ default:
+ throw new UnsupportedOperationException(operatorType.toString() + " is not supported");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ProcessLogicalOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ProcessLogicalOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ProcessLogicalOperator.java
new file mode 100644
index 0000000..395eb65
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ProcessLogicalOperator.java
@@ -0,0 +1,45 @@
+package com.jwplayer.sqe.trident.function;
+
+import com.jwplayer.sqe.language.expression.transform.predicate.LogicalOperatorType;
+import org.apache.storm.trident.operation.BaseFunction;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Values;
+
+
+public class ProcessLogicalOperator extends BaseFunction {
+ private LogicalOperatorType operatorType = null;
+
+ public ProcessLogicalOperator(LogicalOperatorType operatorType) {
+ this.operatorType = operatorType;
+ }
+
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ Boolean retVal = tuple.getBoolean(0);
+
+ switch(operatorType) {
+ case And:
+ for(int i = 1; i < tuple.size(); i++) {
+ retVal = retVal && tuple.getBoolean(i);
+ }
+ break;
+ case Not:
+ retVal = !retVal;
+ break;
+ case Or:
+ for(int i = 1; i < tuple.size(); i++) {
+ retVal = retVal || tuple.getBoolean(i);
+ }
+ break;
+ case Xor:
+ for(int i = 1; i < tuple.size(); i++) {
+ retVal = retVal ^ tuple.getBoolean(i);
+ }
+ break;
+ default:
+ throw new UnsupportedOperationException(operatorType.toString() + " is not supported");
+ }
+
+ collector.emit(new Values(retVal));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/RoundDate.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/RoundDate.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/RoundDate.java
new file mode 100644
index 0000000..780efb3
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/RoundDate.java
@@ -0,0 +1,44 @@
+package com.jwplayer.sqe.trident.function;
+
+import org.apache.storm.trident.operation.BaseFunction;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.tuple.TridentTuple;
+import org.apache.storm.tuple.Values;
+
+import java.util.Date;
+
+
+public class RoundDate extends BaseFunction {
+ @Override
+ public void execute(TridentTuple tuple, TridentCollector collector) {
+ try {
+ Date date = (Date) tuple.get(0);
+ Integer amount = ((Number) tuple.get(1)).intValue();
+ String unit = tuple.getString(2);
+ Long time = date.getTime();
+ Long adjustment;
+
+ switch(unit.toLowerCase()) {
+ case "second":
+ adjustment = 1000l * amount;
+ break;
+ case "minute":
+ adjustment = 60000l * amount;
+ break;
+ case "hour":
+ adjustment = 3600000l * amount;
+ break;
+ case "day":
+ adjustment = 86400000l * amount;
+ break;
+ default:
+ throw new RuntimeException(unit + " is not a valid unit");
+ }
+
+ time = time / adjustment * adjustment;
+ collector.emit(new Values(new Date(time)));
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredOpaqueTridentKafkaSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredOpaqueTridentKafkaSpout.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredOpaqueTridentKafkaSpout.java
new file mode 100644
index 0000000..9f01628
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredOpaqueTridentKafkaSpout.java
@@ -0,0 +1,45 @@
+package com.jwplayer.sqe.trident.spout.kafka;
+
+import org.apache.storm.kafka.Partition;
+import org.apache.storm.kafka.trident.GlobalPartitionInformation;
+import org.apache.storm.kafka.trident.OpaqueTridentKafkaSpout;
+import org.apache.storm.kafka.trident.TridentKafkaConfig;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.trident.spout.IOpaquePartitionedTridentSpout;
+import org.apache.storm.tuple.Fields;
+
+import java.util.List;
+import java.util.Map;
+
+
+public class FilteredOpaqueTridentKafkaSpout implements IOpaquePartitionedTridentSpout<List<GlobalPartitionInformation>, Partition, Map> {
+ private TridentKafkaConfig config;
+ private long hwmTtl;
+ private OpaqueTridentKafkaSpout spout;
+
+ public FilteredOpaqueTridentKafkaSpout(TridentKafkaConfig config, long hwmTtl) {
+ this.config = config;
+ this.hwmTtl = hwmTtl;
+ this.spout = new OpaqueTridentKafkaSpout(config);
+ }
+
+ @Override
+ public Map<String, Object> getComponentConfiguration() {
+ return spout.getComponentConfiguration();
+ }
+
+ @Override
+ public Coordinator getCoordinator(Map map, TopologyContext topologyContext) {
+ return spout.getCoordinator(map, topologyContext);
+ }
+
+ @Override
+ public Emitter<List<GlobalPartitionInformation>, Partition, Map> getEmitter(Map conf, TopologyContext context) {
+ return (new FilteredTridentKafkaEmitter(conf, context, config, context.getStormId(), hwmTtl)).asOpaqueEmitter();
+ }
+
+ @Override
+ public Fields getOutputFields() {
+ return spout.getOutputFields();
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTransactionalTridentKafkaSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTransactionalTridentKafkaSpout.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTransactionalTridentKafkaSpout.java
new file mode 100644
index 0000000..71e9cca
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTransactionalTridentKafkaSpout.java
@@ -0,0 +1,46 @@
+package com.jwplayer.sqe.trident.spout.kafka;
+
+import org.apache.storm.kafka.Partition;
+import org.apache.storm.kafka.trident.GlobalPartitionInformation;
+import org.apache.storm.kafka.trident.TransactionalTridentKafkaSpout;
+import org.apache.storm.kafka.trident.TridentKafkaConfig;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.trident.spout.IPartitionedTridentSpout;
+import org.apache.storm.tuple.Fields;
+
+import java.util.Map;
+
+
+public class FilteredTransactionalTridentKafkaSpout implements IPartitionedTridentSpout<GlobalPartitionInformation, Partition, Map> {
+ private TridentKafkaConfig config;
+ private long hwmTtl;
+ private TransactionalTridentKafkaSpout spout;
+
+ public FilteredTransactionalTridentKafkaSpout(TridentKafkaConfig config, long hwmTtl) {
+ this.config = config;
+ this.hwmTtl = hwmTtl;
+ this.spout = new TransactionalTridentKafkaSpout(config);
+ }
+
+ @Override
+ public Map<String, Object> getComponentConfiguration() {
+ return spout.getComponentConfiguration();
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Coordinator<GlobalPartitionInformation> getCoordinator(Map conf, TopologyContext context) {
+ return spout.getCoordinator(conf, context);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Emitter<GlobalPartitionInformation, Partition, Map> getEmitter(Map conf, TopologyContext context) {
+ return (new FilteredTridentKafkaEmitter(conf, context, config, context.getStormId(), hwmTtl)).asTransactionalEmitter();
+ }
+
+ @Override
+ public Fields getOutputFields() {
+ return spout.getOutputFields();
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentCollector.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentCollector.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentCollector.java
new file mode 100644
index 0000000..6ad7174
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentCollector.java
@@ -0,0 +1,88 @@
+package com.jwplayer.sqe.trident.spout.kafka;
+
+import com.jwplayer.sqe.trident.StreamMetadata;
+import org.apache.storm.trident.operation.TridentCollector;
+
+import java.util.*;
+
+
+public class FilteredTridentCollector implements TridentCollector {
+ public static final String HIGH_WATERMARKS = "highWatermarks";
+ public static final String TIMESTAMP = "timestamp";
+ public static final String WATERMARK = "watermark";
+ private TridentCollector collector;
+ private Map<String, Map<String, Long>> hwmMetadata;
+ private int keyIndex;
+ private long hwmTtl;
+
+ @SuppressWarnings("unchecked")
+ public FilteredTridentCollector(TridentCollector collector, int keyIndex, long hwmTtl, Map lastMetadata) {
+ this.collector = collector;
+ this.keyIndex = keyIndex;
+ this.hwmTtl = hwmTtl;
+ if (lastMetadata != null && lastMetadata.containsKey(HIGH_WATERMARKS)) {
+ this.hwmMetadata = (Map) lastMetadata.get(HIGH_WATERMARKS);
+ } else {
+ this.hwmMetadata = new HashMap<>();
+ }
+ }
+
+ @Override
+ public void emit(List<Object> values) {
+ // This is where the magic happens
+ // 1) Pull the key (if it exists, do we have access to it?)
+ // 2) Determine HWM metadata for this set of values
+ // 3) Emit values if this HWM > the previous HWM for the pid + partition
+ // 4) Update HWM metadata if values were emitted
+ byte[] key = (byte[]) values.get(keyIndex);
+
+ if (key != null && key.length == 20) {
+ StreamMetadata streamMetadata = StreamMetadata.parseBytes(key);
+ String pidAndPartition = streamMetadata.getPidAndPartitionAsHex();
+ Long watermark = streamMetadata.offset;
+
+ if (hwmMetadata.containsKey(pidAndPartition)) {
+ Map<String, Long> highWatermark = hwmMetadata.get(pidAndPartition);
+ if (watermark > highWatermark.get(WATERMARK)) {
+ collector.emit(values);
+ highWatermark.put(WATERMARK, watermark);
+ highWatermark.put(TIMESTAMP, System.currentTimeMillis() / 1000L);
+ hwmMetadata.put(pidAndPartition, highWatermark);
+ }
+ } else {
+ collector.emit(values);
+ Map<String, Long> highWatermark = new HashMap<>(2);
+ highWatermark.put(WATERMARK, watermark);
+ highWatermark.put(TIMESTAMP, System.currentTimeMillis() / 1000L);
+ hwmMetadata.put(pidAndPartition, highWatermark);
+ }
+ } else {
+ // Emit messages without parseable keys. These can be replays.
+ collector.emit(values);
+ }
+ }
+
+ @Override
+ public void reportError(Throwable throwable) {
+ collector.reportError(throwable);
+ }
+
+ @SuppressWarnings("unchecked")
+ public Map resolveMetadata(Map newMetadata) {
+ // newMetadata is the metadata generated by the emitter. We need to
+ // resolve the HWM metadata generated in this class with that metadata.
+ // Also remove any high watermarks that have a timestamp old than the TTL.
+ long now = System.currentTimeMillis() / 1000L;
+ Iterator<Map.Entry<String, Map<String, Long>>> iter = hwmMetadata.entrySet().iterator();
+
+ while (iter.hasNext()) {
+ Map.Entry<String, Map<String, Long>> entry = iter.next();
+ if (now - entry.getValue().get(TIMESTAMP) > hwmTtl) {
+ iter.remove();
+ }
+ }
+
+ newMetadata.put(HIGH_WATERMARKS, hwmMetadata);
+ return newMetadata;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentKafkaEmitter.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentKafkaEmitter.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentKafkaEmitter.java
new file mode 100644
index 0000000..d6c5a7a
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/FilteredTridentKafkaEmitter.java
@@ -0,0 +1,101 @@
+package com.jwplayer.sqe.trident.spout.kafka;
+
+import org.apache.storm.kafka.Partition;
+import org.apache.storm.kafka.trident.GlobalPartitionInformation;
+import org.apache.storm.kafka.trident.TridentKafkaConfig;
+import org.apache.storm.kafka.trident.TridentKafkaEmitter;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.trident.operation.TridentCollector;
+import org.apache.storm.trident.spout.IOpaquePartitionedTridentSpout;
+import org.apache.storm.trident.spout.IPartitionedTridentSpout;
+import org.apache.storm.trident.topology.TransactionAttempt;
+
+import java.util.List;
+import java.util.Map;
+
+
+public class FilteredTridentKafkaEmitter {
+ private TridentKafkaEmitter emitter;
+ private long hwmTtl;
+
+ public FilteredTridentKafkaEmitter(Map conf, TopologyContext context, TridentKafkaConfig config, String topologyInstanceId, long hwmTtl) {
+ this.emitter = new TridentKafkaEmitter(conf, context, config, topologyInstanceId);
+ this.hwmTtl = hwmTtl;
+ }
+
+ private static class OpaqueEmitter implements IOpaquePartitionedTridentSpout.Emitter<List<GlobalPartitionInformation>, Partition, Map> {
+ private IOpaquePartitionedTridentSpout.Emitter emitter;
+ private long hwmTtl;
+
+ private OpaqueEmitter(TridentKafkaEmitter tridentKafkaEmitter, long hwmTtl) {
+ this.emitter = tridentKafkaEmitter.asOpaqueEmitter();
+ this.hwmTtl = hwmTtl;
+ }
+
+ @SuppressWarnings("unchecked")
+ public Map emitPartitionBatch(TransactionAttempt transactionAttempt, TridentCollector collector, Partition partition, Map lastMetadata) {
+ FilteredTridentCollector filteredCollector = new FilteredTridentCollector(collector, 0, hwmTtl, lastMetadata); // TODO: keyIndex is hardcoded to 0 here
+ Map newMetadata = (Map) emitter.emitPartitionBatch(transactionAttempt, filteredCollector, partition, lastMetadata);
+ return filteredCollector.resolveMetadata(newMetadata);
+ }
+
+ @SuppressWarnings("unchecked")
+ public void refreshPartitions(List<Partition> partitions) {
+ emitter.refreshPartitions(partitions);
+ }
+
+ @SuppressWarnings("unchecked")
+ public List<Partition> getOrderedPartitions(List<GlobalPartitionInformation> partitionInformation) {
+ return emitter.getOrderedPartitions(partitionInformation);
+ }
+
+ public void close() {
+ emitter.close();
+ }
+ }
+
+ public IOpaquePartitionedTridentSpout.Emitter<List<GlobalPartitionInformation>, Partition, Map> asOpaqueEmitter() {
+ return new OpaqueEmitter(emitter, hwmTtl);
+ }
+
+ private static class TransactionalEmitter implements IPartitionedTridentSpout.Emitter<List<GlobalPartitionInformation>, Partition, Map> {
+ private IPartitionedTridentSpout.Emitter emitter;
+ private long hwmTtl;
+
+ private TransactionalEmitter(TridentKafkaEmitter tridentKafkaEmitter, long hwmTtl) {
+ this.emitter = tridentKafkaEmitter.asTransactionalEmitter();
+ this.hwmTtl = hwmTtl;
+ }
+
+ @SuppressWarnings("unchecked")
+ public Map emitPartitionBatchNew(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map lastMetadata) {
+ FilteredTridentCollector filteredCollector = new FilteredTridentCollector(collector, 0, hwmTtl, lastMetadata); // TODO: keyIndex is hardcoded to 0 here
+ Map newMetadata = (Map) emitter.emitPartitionBatchNew(attempt, collector, partition, lastMetadata);
+ return filteredCollector.resolveMetadata(newMetadata);
+ }
+
+ @SuppressWarnings("unchecked")
+ public void emitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map lastMetadata) {
+ FilteredTridentCollector filteredCollector = new FilteredTridentCollector(collector, 0, hwmTtl, lastMetadata); // TODO: keyIndex is hardcoded to 0 here
+ emitter.emitPartitionBatch(attempt, filteredCollector, partition, lastMetadata);
+ }
+
+ @SuppressWarnings("unchecked")
+ public void refreshPartitions(List<Partition> partitions) {
+ emitter.refreshPartitions(partitions);
+ }
+
+ @SuppressWarnings("unchecked")
+ public List<Partition> getOrderedPartitions(List<GlobalPartitionInformation> partitionInformation) {
+ return emitter.getOrderedPartitions(partitionInformation);
+ }
+
+ public void close() {
+ emitter.close();
+ }
+ }
+
+ public IPartitionedTridentSpout.Emitter asTransactionalEmitter() {
+ return new TransactionalEmitter(emitter, hwmTtl);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/SqeRawFullScheme.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/SqeRawFullScheme.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/SqeRawFullScheme.java
new file mode 100644
index 0000000..538176c
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/spout/kafka/SqeRawFullScheme.java
@@ -0,0 +1,38 @@
+package com.jwplayer.sqe.trident.spout.kafka;
+
+import com.jwplayer.sqe.language.stream.StreamAdapter;
+import com.jwplayer.sqe.trident.StreamMetadata;
+import org.apache.storm.kafka.FullScheme;
+import org.apache.storm.kafka.Partition;
+import org.apache.storm.kafka.ZkHosts;
+import org.apache.storm.spout.RawScheme;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Values;
+import org.apache.storm.utils.Utils;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+
+public class SqeRawFullScheme extends RawScheme implements FullScheme {
+ public static final String KEY_FIELD = "_key";
+ public static final String VALUE_FIELD = "_value";
+ public final long pid;
+
+ public SqeRawFullScheme(String topologyName, String streamName, ZkHosts zkHosts) {
+ pid = StreamAdapter.createPid(topologyName, streamName, zkHosts.brokerZkStr + zkHosts.brokerZkPath);
+ }
+
+ @Override
+ public List<Object> deserialize(ByteBuffer key, ByteBuffer value, Partition partition, long offset) {
+ byte[] keyBytes = key == null ? null : Utils.toByteArray(key);
+ byte[] valueBytes = value == null ? null : Utils.toByteArray(value);
+
+ return new Values(keyBytes, valueBytes, new StreamMetadata(pid, partition.partition, offset));
+ }
+
+ @Override
+ public Fields getOutputFields() {
+ return new Fields(KEY_FIELD, VALUE_FIELD, StreamAdapter.STREAM_METADATA_FIELD);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/BaseGsonSerializer.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/BaseGsonSerializer.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/BaseGsonSerializer.java
new file mode 100644
index 0000000..0a999f1
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/BaseGsonSerializer.java
@@ -0,0 +1,74 @@
+package com.jwplayer.sqe.trident.state;
+
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonNull;
+import com.google.gson.JsonPrimitive;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.storm.trident.state.Serializer;
+
+public abstract class BaseGsonSerializer<T> implements Serializer<T> {
+ public static JsonArray serializeType(Object value) {
+ JsonArray jsonArray = new JsonArray();
+
+ if(value == null) {
+ jsonArray.add(new JsonPrimitive("N"));
+ jsonArray.add(JsonNull.INSTANCE);
+ } else if(value instanceof Integer) {
+ jsonArray.add(new JsonPrimitive("I"));
+ jsonArray.add(new JsonPrimitive((Integer) value));
+ } else if(value instanceof Long) {
+ jsonArray.add(new JsonPrimitive("L"));
+ jsonArray.add(new JsonPrimitive((Long) value));
+ } else if(value instanceof Float) {
+ jsonArray.add(new JsonPrimitive("F"));
+ jsonArray.add(new JsonPrimitive((Float) value));
+ } else if(value instanceof Double) {
+ jsonArray.add(new JsonPrimitive("D"));
+ jsonArray.add(new JsonPrimitive((Double) value));
+ } else if(value instanceof String) {
+ jsonArray.add(new JsonPrimitive("S"));
+ jsonArray.add(new JsonPrimitive((String) value));
+ } else if(value instanceof byte[]) {
+ jsonArray.add(new JsonPrimitive("B"));
+ jsonArray.add(new JsonPrimitive(Base64.encodeBase64String((byte[]) value)));
+ } else {
+ throw new RuntimeException(value.getClass().getName() + "is not a supported type");
+ }
+
+ return jsonArray;
+ }
+
+ public static Object deserializeType(JsonArray jsonArray, int typeIndex, int valueIndex) {
+ String type = jsonArray.get(typeIndex).getAsString();
+ Object value;
+
+ switch(type) {
+ case "N":
+ value = null;
+ break;
+ case "I":
+ value = jsonArray.get(valueIndex).getAsInt();
+ break;
+ case "L":
+ value = jsonArray.get(valueIndex).getAsLong();
+ break;
+ case "F":
+ value = jsonArray.get(valueIndex).getAsFloat();
+ break;
+ case "D":
+ value = jsonArray.get(valueIndex).getAsDouble();
+ break;
+ case "S":
+ value = jsonArray.get(valueIndex).getAsString();
+ break;
+ case "B":
+ value = Base64.decodeBase64(jsonArray.get(valueIndex).getAsString());
+ break;
+ default:
+ throw new RuntimeException(type + " is not a supported type");
+ }
+
+ return value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/GsonOpaqueSerializer.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/GsonOpaqueSerializer.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/GsonOpaqueSerializer.java
new file mode 100644
index 0000000..ae8863f
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/GsonOpaqueSerializer.java
@@ -0,0 +1,46 @@
+package com.jwplayer.sqe.trident.state;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonPrimitive;
+import org.apache.storm.trident.state.OpaqueValue;
+
+import java.io.UnsupportedEncodingException;
+
+
+public class GsonOpaqueSerializer extends BaseGsonSerializer<OpaqueValue> {
+ public byte[] serialize(OpaqueValue value) {
+ try {
+ Gson gson = new Gson();
+ JsonArray jsonArray = new JsonArray();
+ JsonPrimitive txID = new JsonPrimitive(value.getCurrTxid());
+ jsonArray.add(txID);
+ jsonArray.addAll(serializeType(value.getCurr()));
+ jsonArray.addAll(serializeType(value.getPrev()));
+
+ return gson.toJson(jsonArray).getBytes("UTF-8");
+ } catch(UnsupportedEncodingException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public OpaqueValue deserialize(byte[] bytes) {
+ try {
+ Gson gson = new Gson();
+ String json = new String(bytes, "UTF-8");
+ JsonArray jsonArray = gson.fromJson(json, JsonArray.class);
+ Long txID = jsonArray.get(0).getAsLong();
+
+ if(jsonArray.size() == 3) {
+ return new OpaqueValue(txID, deserializeType(jsonArray, 1, 2));
+ }
+ else {
+ return new OpaqueValue(txID, deserializeType(jsonArray, 1, 2), deserializeType(jsonArray, 3, 4));
+ }
+ } catch(UnsupportedEncodingException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/GsonTransactionalSerializer.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/GsonTransactionalSerializer.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/GsonTransactionalSerializer.java
new file mode 100644
index 0000000..c522529
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/GsonTransactionalSerializer.java
@@ -0,0 +1,38 @@
+package com.jwplayer.sqe.trident.state;
+
+import com.google.gson.*;
+import org.apache.storm.trident.state.TransactionalValue;
+
+import java.io.UnsupportedEncodingException;
+
+
+public class GsonTransactionalSerializer extends BaseGsonSerializer<TransactionalValue> {
+ public byte[] serialize(TransactionalValue tValue) {
+ try {
+ Gson gson = new Gson();
+ JsonArray jsonArray = new JsonArray();
+ JsonPrimitive txID = new JsonPrimitive(tValue.getTxid());
+ jsonArray.add(txID);
+ jsonArray.addAll(serializeType(tValue.getVal()));
+
+ return gson.toJson(jsonArray).getBytes("UTF-8");
+ } catch(UnsupportedEncodingException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public TransactionalValue deserialize(byte[] bytes) {
+ try {
+ Gson gson = new Gson();
+ String json = new String(bytes, "UTF-8");
+ JsonArray jsonArray = gson.fromJson(json, JsonArray.class);
+ Long txID = jsonArray.get(0).getAsLong();
+
+ return new TransactionalValue(txID, deserializeType(jsonArray, 1, 2));
+ } catch(UnsupportedEncodingException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/mongodb/MongoDBState.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/mongodb/MongoDBState.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/mongodb/MongoDBState.java
new file mode 100644
index 0000000..3bd4e3c
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/mongodb/MongoDBState.java
@@ -0,0 +1,228 @@
+package com.jwplayer.sqe.trident.state.mongodb;
+
+import com.clearspring.analytics.hash.MurmurHash;
+import com.mongodb.*;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.UpdateOneModel;
+import com.mongodb.client.model.UpdateOptions;
+import org.bson.Document;
+import org.apache.storm.task.IMetricsContext;
+import org.apache.storm.trident.state.*;
+import org.apache.storm.trident.state.map.*;
+import org.apache.storm.tuple.Values;
+
+import java.io.ByteArrayOutputStream;
+import java.util.*;
+
+
+import static com.mongodb.client.model.Filters.*;
+
+public class MongoDBState<T> implements IBackingMap<T> {
+ private MongoClient mongoClient = null;
+ private MongoCollection mongoCollection = null;
+ private MongoDatabase mongoDB = null;
+ private String collectionName;
+ private List<String> keyFields;
+ private String valueField;
+ private MongoDBStateOptions options;
+ private StateType stateType;
+
+ public MongoDBState(String collectionName, List<String> keyFields, String valueField, MongoDBStateOptions options, StateType stateType) {
+ this.collectionName = collectionName;
+ this.keyFields = keyFields;
+ this.valueField = valueField;
+ this.options = options;
+ this.stateType = stateType;
+ }
+
+ public static StateFactory nonTransactional(String collectionName, List<String> keyFields, String valueField, MongoDBStateOptions options) {
+ return new Factory(collectionName, keyFields, valueField, options, StateType.NON_TRANSACTIONAL);
+ }
+
+ public static StateFactory opaque(String collectionName, List<String> keyFields, String valueField, MongoDBStateOptions options) {
+ return new Factory(collectionName, keyFields, valueField, options, StateType.OPAQUE);
+ }
+
+ public static StateFactory transactional(String collectionName, List<String> keyFields, String valueField, MongoDBStateOptions options) {
+ return new Factory(collectionName, keyFields, valueField, options, StateType.TRANSACTIONAL);
+ }
+
+ protected static class Factory implements StateFactory {
+ private String collectionName;
+ private List<String> keyFields;
+ private MongoDBStateOptions options;
+ private StateType stateType;
+ private String valueField;
+
+ public Factory(String collectionName, List<String> keyFields, String valueField, MongoDBStateOptions options, StateType stateType) {
+ this.collectionName = collectionName;
+ this.keyFields = keyFields;
+ this.options = options;
+ this.stateType = stateType;
+ this.valueField = valueField;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public State makeState(Map map, IMetricsContext context, int partitionIndex, int numPartitions) {
+ MongoDBState mongoDBState = new MongoDBState(collectionName, keyFields, valueField, options, stateType);
+ CachedMap cachedMap = new CachedMap(mongoDBState, options.cacheSize);
+ MapState mapState;
+
+ switch(stateType) {
+ case NON_TRANSACTIONAL:
+ mapState = NonTransactionalMap.build(cachedMap);
+ break;
+ case TRANSACTIONAL:
+ mapState = TransactionalMap.build(cachedMap);
+ break;
+ case OPAQUE:
+ mapState = OpaqueMap.build(cachedMap);
+ break;
+ default:
+ throw new RuntimeException("Unknown state type: " + stateType);
+ }
+
+ return new SnapshottableMap(mapState, new Values(options.globalKey));
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private T deserializeValue(Document value) {
+ if(value == null) return null;
+
+ // We have to handle the BSON binary value and convert it to byte[]
+ Object internalValue = value.get(options.valueName);
+ if(internalValue instanceof org.bson.types.Binary) internalValue = ((org.bson.types.Binary) internalValue).getData();
+
+ switch (stateType) {
+ case NON_TRANSACTIONAL:
+ return (T) internalValue;
+ case TRANSACTIONAL:
+ return (T) new TransactionalValue<>(value.getLong(options.txIdName), internalValue);
+ case OPAQUE:
+ Object prevInternalValue = value.get(options.prevValueName);
+ if(prevInternalValue instanceof org.bson.types.Binary) prevInternalValue = ((org.bson.types.Binary) prevInternalValue).getData();
+
+ return (T) new OpaqueValue<>(value.getLong(options.txIdName), internalValue, prevInternalValue);
+ default:
+ return null;
+ }
+ }
+
+ private Long getID(List<Object> keys) {
+ ByteArrayOutputStream stream = new ByteArrayOutputStream();
+
+ try {
+ for (Object key : keys) {
+ stream.write(String.valueOf(key).getBytes());
+ }
+ stream.write(valueField.getBytes());
+ } catch (Exception ex) {
+ throw new RuntimeException("Could not create _id from given keys", ex);
+ }
+
+ byte[] bytes = stream.toByteArray();
+ return MurmurHash.hash64(bytes, bytes.length);
+ }
+
+ private MongoClient getMongoClient() {
+ if (mongoClient == null) {
+ MongoClientOptions clientOptions = MongoClientOptions.builder()
+ .readPreference(ReadPreference.primaryPreferred())
+ .requiredReplicaSetName(options.replicaSet)
+ .build();
+ List<ServerAddress> servers = new ArrayList<>();
+ for(String host: options.hosts) {
+ servers.add(new ServerAddress(host, options.port));
+ }
+ mongoClient = new MongoClient(
+ servers,
+ Collections.singletonList(MongoCredential.createScramSha1Credential(
+ options.userName,
+ options.db,
+ options.password.toCharArray())),
+ clientOptions);
+ }
+
+ return mongoClient;
+ }
+
+ @SuppressWarnings("unchecked")
+ private MongoCollection<Document> getMongoCollection() {
+ if (mongoCollection == null) {
+ mongoCollection = getMongoDatabase().getCollection(collectionName);
+ }
+
+ return mongoCollection;
+ }
+
+ private MongoDatabase getMongoDatabase() {
+ if (mongoDB == null) {
+ mongoDB = getMongoClient().getDatabase(options.db);
+ }
+
+ return mongoDB;
+ }
+
+ private Document serializeValue(T value) {
+ switch (stateType) {
+ case NON_TRANSACTIONAL:
+ return new Document(options.valueName, value);
+ case TRANSACTIONAL:
+ Map<String, Object> tMap = new HashMap<>();
+ tMap.put(options.txIdName, ((TransactionalValue) value).getTxid());
+ tMap.put(options.valueName, ((TransactionalValue) value).getVal());
+ return new Document(tMap);
+ case OPAQUE:
+ Map<String, Object> oMap = new HashMap<>();
+ oMap.put(options.txIdName, ((OpaqueValue) value).getCurrTxid());
+ oMap.put(options.valueName, ((OpaqueValue) value).getCurr());
+ oMap.put(options.prevValueName, ((OpaqueValue) value).getPrev());
+ return new Document(oMap);
+ default:
+ return null;
+ }
+ }
+
+ @Override
+ public List<T> multiGet(List<List<Object>> keysList) {
+ MongoCollection collection = getMongoCollection();
+ List<T> retVal = new ArrayList<>();
+
+ for(List<Object> keys: keysList) {
+ Document document = (Document) collection.find(eq("_id", getID(keys))).first();
+ if(document == null) {
+ retVal.add(null);
+ }
+ else {
+ retVal.add(deserializeValue((Document) document.get(valueField)));
+ }
+ }
+
+ return retVal;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void multiPut(List<List<Object>> keysList, List<T> values) {
+ MongoCollection collection = getMongoCollection();
+ List<UpdateOneModel> updateOneModels = new ArrayList<>();
+ UpdateOptions updateOptions = new UpdateOptions().upsert(true);
+
+ for(int i = 0; i < keysList.size(); i++) {
+ Map<String, Object> map = new HashMap<>();
+ for(int j = 0; j < keysList.get(i).size(); j++) {
+ map.put(keyFields.get(j), keysList.get(i).get(j));
+ }
+ map.put(valueField, serializeValue(values.get(i)));
+ updateOneModels.add(new UpdateOneModel(
+ eq("_id", getID(keysList.get(i))),
+ new Document("$set", new Document(map)),
+ updateOptions));
+ }
+
+ collection.bulkWrite(updateOneModels);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/mongodb/MongoDBStateOptions.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/mongodb/MongoDBStateOptions.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/mongodb/MongoDBStateOptions.java
new file mode 100644
index 0000000..ba6643d
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/mongodb/MongoDBStateOptions.java
@@ -0,0 +1,35 @@
+package com.jwplayer.sqe.trident.state.mongodb;
+
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+public class MongoDBStateOptions implements Serializable {
+ public int cacheSize = 5000;
+ public String db = "";
+ public String globalKey = "$MONGO-DB-STATE-GLOBAL";
+ public List<String> hosts = null;
+ public String password = "";
+ public int port = 27017;
+ public String prevValueName = "Prev";
+ public String replicaSet = "";
+ public String txIdName = "TxId";
+ public String userName = "";
+ public String valueName = "Value";
+
+ @SuppressWarnings("unchecked")
+ public static MongoDBStateOptions parse(Map map) {
+ MongoDBStateOptions options = new MongoDBStateOptions();
+
+ if(map.containsKey("jw.sqe.state.mongodb.cachesize")) options.cacheSize = (int) map.get("jw.sqe.state.mongodb.cachesize");
+ if(map.containsKey("jw.sqe.state.mongodb.db")) options.db = (String) map.get("jw.sqe.state.mongodb.db");
+ if(map.containsKey("jw.sqe.state.mongodb.hosts")) options.hosts = (List<String>) map.get("jw.sqe.state.mongodb.hosts");
+ if(map.containsKey("jw.sqe.state.mongodb.password")) options.password = (String) map.get("jw.sqe.state.mongodb.password");
+ if(map.containsKey("jw.sqe.state.mongodb.port")) options.port = Integer.parseInt((String) map.get("jw.sqe.state.mongodb.port"));
+ if(map.containsKey("jw.sqe.state.mongodb.replicaSet")) options.replicaSet = (String) map.get("jw.sqe.state.mongodb.replicaSet");
+ if(map.containsKey("jw.sqe.state.mongodb.userName")) options.userName = (String) map.get("jw.sqe.state.mongodb.userName");
+
+ return options;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/redis/RedisKeyFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/redis/RedisKeyFactory.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/redis/RedisKeyFactory.java
new file mode 100644
index 0000000..42881a9
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/state/redis/RedisKeyFactory.java
@@ -0,0 +1,47 @@
+package com.jwplayer.sqe.trident.state.redis;
+
+import org.apache.storm.redis.trident.state.KeyFactory;
+
+import java.util.List;
+
+public class RedisKeyFactory implements KeyFactory {
+ public String delimiter;
+ public List<Integer> keyIndexes;
+ public String prefix;
+ public String suffix;
+
+ public RedisKeyFactory(String delimiter, List<Integer> keyIndexes, String prefix, String suffix) {
+ this.delimiter = delimiter;
+ this.keyIndexes = keyIndexes;
+ this.prefix = prefix;
+ this.suffix = suffix;
+ }
+
+ @Override
+ public String build(List<Object> keys) {
+ StringBuilder sb = new StringBuilder();
+
+ // Append prefix, if it exists
+ if(prefix != null && !prefix.equals("")) {
+ sb.append(prefix);
+ if(keyIndexes.size() > 0 || (suffix != null && !suffix.equals(""))) sb.append(delimiter);
+ }
+
+ // Append the keys we want based on their indexes
+ for(int i = 0; i < keyIndexes.size() - 1; ++i) {
+ sb.append(keys.get(keyIndexes.get(i)));
+ sb.append(delimiter);
+ }
+ if(keyIndexes.size() > 0){
+ sb.append(keys.get(keyIndexes.get(keyIndexes.size() - 1)));
+ }
+
+ // Append the suffix, if it exists
+ if(suffix != null && !suffix.equals("")) {
+ if(keyIndexes.size() > 0 || (prefix != null && !prefix.equals(""))) sb.append(delimiter);
+ sb.append(suffix);
+ }
+
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/FileHelper.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/FileHelper.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/FileHelper.java
new file mode 100644
index 0000000..6e8d7ec
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/FileHelper.java
@@ -0,0 +1,44 @@
+package com.jwplayer.sqe.util;
+
+import org.apache.commons.io.IOUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+
+public class FileHelper {
+ public static InputStream getInputStream(URI uri) throws IOException, URISyntaxException {
+ URI fsURI;
+ // This is a bit hacky because of weird behavior getting the file system.
+ // We also want relative URIs to resolve to the default/local file system.
+ if (uri.isAbsolute()) {
+ fsURI = new URI(uri.getScheme(), uri.getAuthority() == null ? "" : uri.getAuthority(), "/", null, null);
+ } else {
+ fsURI = new URI("file", uri.getAuthority() == null ? "" : uri.getAuthority(), "/", null, null);
+ }
+ FileSystem fs = FileSystems.getFileSystem(fsURI);
+ Path path;
+
+ // Windows paths can have a preceding slash that breaks this, even if Java gave it to us! :D
+ if (
+ System.getProperty("os.name").contains("indow")
+ && uri.getPath().substring(0, 1).equals("/")
+ && fsURI.getScheme().equals("file")) {
+ path = fs.getPath(uri.getPath().substring(1));
+ } else {
+ path = fs.getPath(uri.getPath());
+ }
+
+ return Files.newInputStream(path);
+ }
+
+ public static String loadFileAsString(URI uri) throws IOException, URISyntaxException {
+ return IOUtils.toString(getInputStream(uri));
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/3654b8ca/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/YamlConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/YamlConfig.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/YamlConfig.java
new file mode 100644
index 0000000..deaef02
--- /dev/null
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/util/YamlConfig.java
@@ -0,0 +1,149 @@
+package com.jwplayer.sqe.util;
+
+import org.apache.log4j.Logger;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+public class YamlConfig {
+ public static final String CONFIG_PATH = "conf/sample-conf.yaml";
+
+ public static Yaml getJWConfig(String configPath) throws IOException, URISyntaxException {
+ if (configPath == null) {
+ configPath = CONFIG_PATH;
+ }
+
+ Yaml config = new Yaml();
+ config.load(FileHelper.getInputStream(new URI(configPath)));
+ return config;
+ }
+
+ public static Yaml getJWConfig() throws IOException, URISyntaxException {
+ return getJWConfig(null);
+ }
+
+ public static Map<String, Object> getJWConfigAsMap(String configPath) throws IOException, URISyntaxException {
+ if (configPath == null) {
+ configPath = CONFIG_PATH;
+ }
+
+ Yaml config = new Yaml();
+ return (HashMap<String, Object>) config.load(FileHelper.getInputStream(new URI(configPath)));
+ }
+
+ public static Map<String, Object> getJWConfigAsMap() throws IOException, URISyntaxException {
+ return getJWConfigAsMap(null);
+ }
+
+ public static InnerConfig getConfig(String configPath) {
+ return new InnerConfig(configPath);
+ }
+
+ public static InnerConfig getConfig() {
+ return new InnerConfig(CONFIG_PATH);
+ }
+
+
+ public static class InnerConfig {
+
+ Map<String, Object> config;
+
+ public InnerConfig(String path) {
+ try {
+ config = YamlConfig.getJWConfigAsMap(path);
+ } catch (IOException|URISyntaxException e) {
+ throw new RuntimeException("No dice!", e);
+ }
+ }
+
+ public InnerConfig(Map<String, Object> config) {
+ this.config = config;
+ }
+
+ public Object get(String key, Object defaultValue) {
+ try {
+ if (config.get(key) == null)
+ return defaultValue;
+ return config.get(key);
+ } catch (Exception e) {
+ return defaultValue;
+ }
+ }
+
+ public String getString(String key, String defaultValue){
+ try {
+ if (config.get(key) == null)
+ return defaultValue;
+ return (String) config.get(key);
+ } catch (Exception e) {
+ return defaultValue;
+ }
+ }
+
+ public Map<String, Object> getMap(String key, Map<String, Object> defaultValue){
+ try {
+ if (config.get(key) == null) {
+ return defaultValue;
+ }
+ return (Map<String, Object>) config.get(key);
+ } catch (Exception ex) {
+ return defaultValue;
+ }
+ }
+
+ public boolean getBoolean(String key, boolean defaultValue) {
+ try {
+ if (config.get(key) == null) {
+ return defaultValue;
+ }
+ return (Boolean) config.get(key);
+ } catch (Exception ex) {
+ return defaultValue;
+ }
+ }
+
+ public InnerConfig getInnerConfig(String key, Map<String, Object> defaultValue) {
+ return new InnerConfig(getMap(key, defaultValue));
+ }
+
+ public int getInteger(String key, int defaultValue) {
+ try {
+ if (config.get(key) == null) {
+ return defaultValue;
+ }
+ return (Integer)config.get(key);
+ } catch (Exception e) {
+ return defaultValue;
+ }
+ }
+
+ public double getDouble(String key, double defaultValue) {
+ try {
+ if (config.get(key) == null) {
+ return defaultValue;
+ }
+ return (Double) config.get(key);
+ } catch (Exception e) {
+ return defaultValue;
+ }
+ }
+
+ public List getList(String key, List defaultValue){
+ try {
+ if (config.get(key) == null) {
+ return defaultValue;
+ }
+ return (List) config.get(key);
+ } catch (Exception e) {
+ return defaultValue;
+ }
+ }
+
+ }
+}
[07/11] storm git commit: SQE: Apply Apache license headers
Posted by pt...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LessThanEqualCompareNumbers.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LessThanEqualCompareNumbers.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LessThanEqualCompareNumbers.java
index a8ce47c..8107c55 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LessThanEqualCompareNumbers.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LessThanEqualCompareNumbers.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform.predicate;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LogicalOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LogicalOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LogicalOperator.java
index f1c64aa..59460c7 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LogicalOperator.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LogicalOperator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform.predicate;
import com.jwplayer.sqe.trident.function.ProcessLogicalOperator;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LogicalOperatorType.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LogicalOperatorType.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LogicalOperatorType.java
index cf3e20c..27ee172 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LogicalOperatorType.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/LogicalOperatorType.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform.predicate;
public enum LogicalOperatorType {
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NotEqualCompareNumbers.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NotEqualCompareNumbers.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NotEqualCompareNumbers.java
index f4ca8c6..35f3550 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NotEqualCompareNumbers.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NotEqualCompareNumbers.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform.predicate;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NotLogicalOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NotLogicalOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NotLogicalOperator.java
index 0925144..85ab6ba 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NotLogicalOperator.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NotLogicalOperator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform.predicate;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NumberComparisonType.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NumberComparisonType.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NumberComparisonType.java
index 581cf40..6f801ed 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NumberComparisonType.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/NumberComparisonType.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform.predicate;
public enum NumberComparisonType {
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/OrLogicalOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/OrLogicalOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/OrLogicalOperator.java
index 80e24d3..08d68dd 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/OrLogicalOperator.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/OrLogicalOperator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform.predicate;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/PredicateExpression.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/PredicateExpression.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/PredicateExpression.java
index 8523406..c04dcbe 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/PredicateExpression.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/PredicateExpression.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform.predicate;
import com.jwplayer.sqe.language.expression.transform.TransformExpression;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/RLike.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/RLike.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/RLike.java
index a4debed..63b808b 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/RLike.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/RLike.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform.predicate;
import com.jwplayer.sqe.trident.function.EvaluateRegularExpression;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/XorLogicalOperator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/XorLogicalOperator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/XorLogicalOperator.java
index ae722ce..fe0220e 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/XorLogicalOperator.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/expression/transform/predicate/XorLogicalOperator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.expression.transform.predicate;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/BaseDeserializer.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/BaseDeserializer.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/BaseDeserializer.java
index 720ce2d..83e50b5 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/BaseDeserializer.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/BaseDeserializer.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.serde;
import com.jwplayer.sqe.language.serde.avro.AvroDeserializer;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/IdentityDeserializer.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/IdentityDeserializer.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/IdentityDeserializer.java
index 76fc262..1c9a179 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/IdentityDeserializer.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/IdentityDeserializer.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.serde;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/avro/AvroDeserializer.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/avro/AvroDeserializer.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/avro/AvroDeserializer.java
index f5868f4..05e2300 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/avro/AvroDeserializer.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/avro/AvroDeserializer.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.serde.avro;
import com.jwplayer.sqe.language.serde.BaseDeserializer;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/avro/AvroDeserializerOptions.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/avro/AvroDeserializerOptions.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/avro/AvroDeserializerOptions.java
index 3abd155..0e4aa91 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/avro/AvroDeserializerOptions.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/serde/avro/AvroDeserializerOptions.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.serde.avro;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/StateAdapter.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/StateAdapter.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/StateAdapter.java
index 0483a5b..2026629 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/StateAdapter.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/StateAdapter.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.state;
import com.jwplayer.sqe.language.state.kafka.KafkaStateAdapter;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/StateOperationType.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/StateOperationType.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/StateOperationType.java
index 4b56fd2..881cbe3 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/StateOperationType.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/StateOperationType.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.state;
public enum StateOperationType {
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/JwTridentKafkaStateFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/JwTridentKafkaStateFactory.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/JwTridentKafkaStateFactory.java
index 9cc927e..24fd18f 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/JwTridentKafkaStateFactory.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/JwTridentKafkaStateFactory.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.state.kafka;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/KafkaStateAdapter.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/KafkaStateAdapter.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/KafkaStateAdapter.java
index 71ac874..237f1d3 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/KafkaStateAdapter.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/KafkaStateAdapter.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.state.kafka;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/KafkaStateOptions.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/KafkaStateOptions.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/KafkaStateOptions.java
index 9093c11..5d10b77 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/KafkaStateOptions.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/KafkaStateOptions.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.state.kafka;
import com.google.common.base.Joiner;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/SourcePartitionPartitioner.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/SourcePartitionPartitioner.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/SourcePartitionPartitioner.java
index 38074b2..395b661 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/SourcePartitionPartitioner.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/kafka/SourcePartitionPartitioner.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.state.kafka;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/mongodb/MongoDBStateAdapter.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/mongodb/MongoDBStateAdapter.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/mongodb/MongoDBStateAdapter.java
index 6ef1c88..ce7daa7 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/mongodb/MongoDBStateAdapter.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/mongodb/MongoDBStateAdapter.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.state.mongodb;
import com.jwplayer.sqe.language.state.StateAdapter;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateAdapter.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateAdapter.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateAdapter.java
index 986878b..45940cb 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateAdapter.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateAdapter.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.state.redis;
import com.jwplayer.sqe.trident.state.GsonOpaqueSerializer;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateFactory.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateFactory.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateFactory.java
index 9faa980..b1bcd3b 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateFactory.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateFactory.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.state.redis;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateOptions.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateOptions.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateOptions.java
index 3699480..793eacf 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateOptions.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/state/redis/RedisStateOptions.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.state.redis;
import org.apache.storm.redis.trident.state.RedisDataTypeDescription.RedisDataType;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/InputStreamType.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/InputStreamType.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/InputStreamType.java
index 7bebbd6..cd452d0 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/InputStreamType.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/InputStreamType.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.stream;
public enum InputStreamType {
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/StreamAdapter.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/StreamAdapter.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/StreamAdapter.java
index f43c35a..623b5bd 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/StreamAdapter.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/StreamAdapter.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.stream;
import com.clearspring.analytics.hash.MurmurHash;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/StreamContainer.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/StreamContainer.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/StreamContainer.java
index 8f15ac1..c3a0435 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/StreamContainer.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/StreamContainer.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.stream;
import org.apache.storm.trident.Stream;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/kafka/KafkaStreamAdapter.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/kafka/KafkaStreamAdapter.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/kafka/KafkaStreamAdapter.java
index e3e5c91..4e4ea0b 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/kafka/KafkaStreamAdapter.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/kafka/KafkaStreamAdapter.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.stream.kafka;
import com.jwplayer.sqe.language.stream.StreamAdapter;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/kafka/KafkaStreamAdapterOptions.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/kafka/KafkaStreamAdapterOptions.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/kafka/KafkaStreamAdapterOptions.java
index 2c70053..53b9471 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/kafka/KafkaStreamAdapterOptions.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/kafka/KafkaStreamAdapterOptions.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.stream.kafka;
import com.google.common.base.Joiner;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/testing/FixedBatchSpoutOptions.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/testing/FixedBatchSpoutOptions.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/testing/FixedBatchSpoutOptions.java
index defb4db..8a5f0da 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/testing/FixedBatchSpoutOptions.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/testing/FixedBatchSpoutOptions.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.stream.testing;
import org.apache.storm.tuple.Fields;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/testing/FixedBatchSpoutStreamAdapter.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/testing/FixedBatchSpoutStreamAdapter.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/testing/FixedBatchSpoutStreamAdapter.java
index 310763e..1233c4f 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/testing/FixedBatchSpoutStreamAdapter.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/language/stream/testing/FixedBatchSpoutStreamAdapter.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.language.stream.testing;
import com.jwplayer.sqe.language.stream.StreamAdapter;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/ListValuesCollector.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/ListValuesCollector.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/ListValuesCollector.java
index eb638a3..3b71339 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/ListValuesCollector.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/ListValuesCollector.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident;
import org.apache.storm.trident.operation.TridentCollector;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/SingleValuesCollector.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/SingleValuesCollector.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/SingleValuesCollector.java
index 99b6ad3..39d97c7 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/SingleValuesCollector.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/SingleValuesCollector.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident;
import org.apache.storm.trident.operation.TridentCollector;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/StreamMetadata.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/StreamMetadata.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/StreamMetadata.java
index 913ea63..4c10169 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/StreamMetadata.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/StreamMetadata.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident;
import com.google.common.base.Preconditions;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/ValueFilter.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/ValueFilter.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/ValueFilter.java
index fad4651..dcbb3d0 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/ValueFilter.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/ValueFilter.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident;
import org.apache.storm.trident.operation.BaseFilter;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/CardinalityEstimatorAggregator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/CardinalityEstimatorAggregator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/CardinalityEstimatorAggregator.java
index e106aa4..1314f7a 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/CardinalityEstimatorAggregator.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/CardinalityEstimatorAggregator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.aggregator;
import org.apache.storm.tuple.Values;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/CardinalityEstimatorCombinerAggregator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/CardinalityEstimatorCombinerAggregator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/CardinalityEstimatorCombinerAggregator.java
index 32da432..1647ded 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/CardinalityEstimatorCombinerAggregator.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/CardinalityEstimatorCombinerAggregator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.aggregator;
import org.apache.storm.trident.operation.CombinerAggregator;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/GetMaximumAggregator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/GetMaximumAggregator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/GetMaximumAggregator.java
index 7505b93..1300938 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/GetMaximumAggregator.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/GetMaximumAggregator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.aggregator;
import clojure.lang.Numbers;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllAggregator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllAggregator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllAggregator.java
index 0b62dfc..40e3911 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllAggregator.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllAggregator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.aggregator;
import com.clearspring.analytics.stream.cardinality.HyperLogLog;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllCombinerAggregator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllCombinerAggregator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllCombinerAggregator.java
index ae0ca59..4b2b1b0 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllCombinerAggregator.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllCombinerAggregator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.aggregator;
import com.clearspring.analytics.stream.cardinality.CardinalityMergeException;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllpAggregator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllpAggregator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllpAggregator.java
index 5fbea2f..9b05b0a 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllpAggregator.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllpAggregator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.aggregator;
import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllpCombinerAggregator.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllpCombinerAggregator.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllpCombinerAggregator.java
index 77078c6..e4b9703 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllpCombinerAggregator.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/aggregator/HllpCombinerAggregator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.aggregator;
import com.clearspring.analytics.stream.cardinality.CardinalityMergeException;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/AddField.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/AddField.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/AddField.java
index fb598a2..4ebbf08 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/AddField.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/AddField.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import org.apache.storm.tuple.Values;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/CompareNumbers.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/CompareNumbers.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/CompareNumbers.java
index 8ddb99a..f724372 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/CompareNumbers.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/CompareNumbers.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import clojure.lang.Numbers;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/CompareObjects.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/CompareObjects.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/CompareObjects.java
index 5625c7c..4f49503 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/CompareObjects.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/CompareObjects.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import com.jwplayer.sqe.language.expression.transform.predicate.NumberComparisonType;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ConvertMetadataToBytes.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ConvertMetadataToBytes.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ConvertMetadataToBytes.java
index e5eb525..fd33a37 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ConvertMetadataToBytes.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ConvertMetadataToBytes.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import com.jwplayer.sqe.trident.StreamMetadata;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/EvaluateRegularExpression.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/EvaluateRegularExpression.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/EvaluateRegularExpression.java
index 739879e..06acf03 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/EvaluateRegularExpression.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/EvaluateRegularExpression.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import org.apache.storm.trident.operation.BaseFunction;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ExpandKeys.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ExpandKeys.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ExpandKeys.java
index 4178679..0d0ac44 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ExpandKeys.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ExpandKeys.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import org.apache.avro.util.Utf8;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ExpandValues.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ExpandValues.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ExpandValues.java
index 4a26d4c..055c698 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ExpandValues.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/ExpandValues.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import org.apache.avro.util.Utf8;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/FormatDate.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/FormatDate.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/FormatDate.java
index 6b3762b..d298be5 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/FormatDate.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/FormatDate.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import org.apache.storm.trident.operation.BaseFunction;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetCardinalityEstimation.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetCardinalityEstimation.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetCardinalityEstimation.java
index 9a5a41f..058128a 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetCardinalityEstimation.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetCardinalityEstimation.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import com.clearspring.analytics.stream.cardinality.HyperLogLog;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetIf.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetIf.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetIf.java
index a41d4cf..89c0929 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetIf.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetIf.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import org.apache.storm.tuple.Values;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetMapValue.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetMapValue.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetMapValue.java
index 21e7eba..4f48276 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetMapValue.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetMapValue.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetTime.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetTime.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetTime.java
index ddbb0a3..14e58b4 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetTime.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetTime.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import org.apache.log4j.Logger;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetTupleFromAvro.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetTupleFromAvro.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetTupleFromAvro.java
index 94544e9..8a2b02a 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetTupleFromAvro.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/GetTupleFromAvro.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import org.apache.avro.Schema;
http://git-wip-us.apache.org/repos/asf/storm/blob/a667c75d/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/Hash.java
----------------------------------------------------------------------
diff --git a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/Hash.java b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/Hash.java
index dfbd85e..71cf23e 100644
--- a/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/Hash.java
+++ b/external/storm-sqe/src/main/java/com/jwplayer/sqe/trident/function/Hash.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.jwplayer.sqe.trident.function;
import com.clearspring.analytics.hash.MurmurHash;