You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by mb...@apache.org on 2015/06/15 11:33:00 UTC
[10/27] flink git commit: [storm-compat] Storm compatibility code
cleanup
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-compatibility/src/test/java/org/apache/flink/stormcompatibility/wrappers/StormTupleTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-compatibility/src/test/java/org/apache/flink/stormcompatibility/wrappers/StormTupleTest.java b/flink-staging/flink-streaming/flink-storm-compatibility/src/test/java/org/apache/flink/stormcompatibility/wrappers/StormTupleTest.java
index 133407a..14b1c60 100644
--- a/flink-staging/flink-streaming/flink-storm-compatibility/src/test/java/org/apache/flink/stormcompatibility/wrappers/StormTupleTest.java
+++ b/flink-staging/flink-streaming/flink-storm-compatibility/src/test/java/org/apache/flink/stormcompatibility/wrappers/StormTupleTest.java
@@ -17,337 +17,333 @@
package org.apache.flink.stormcompatibility.wrappers;
-import java.util.List;
-
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.stormcompatibility.util.AbstractTest;
import org.junit.Assert;
import org.junit.Test;
-
-
-
+import java.util.List;
public class StormTupleTest extends AbstractTest {
-
+
@Test
public void nonTupleTest() {
- final Object flinkTuple = new Integer(this.r.nextInt());
-
+ final Object flinkTuple = this.r.nextInt();
+
final StormTuple<Object> tuple = new StormTuple<Object>(flinkTuple);
Assert.assertSame(flinkTuple, tuple.getValue(0));
-
+
final List<Object> values = tuple.getValues();
Assert.assertEquals(1, values.size());
Assert.assertEquals(flinkTuple, values.get(0));
}
-
+
@Test
public void tupleTest() throws InstantiationException, IllegalAccessException {
final int numberOfAttributes = 1 + this.r.nextInt(25);
final Object[] data = new Object[numberOfAttributes];
-
+
final Tuple flinkTuple = Tuple.getTupleClass(numberOfAttributes).newInstance();
- for(int i = 0; i < numberOfAttributes; ++i) {
- data[i] = new Integer(this.r.nextInt());
+ for (int i = 0; i < numberOfAttributes; ++i) {
+ data[i] = this.r.nextInt();
flinkTuple.setField(data[i], i);
}
-
+
final StormTuple<Tuple> tuple = new StormTuple<Tuple>(flinkTuple);
final List<Object> values = tuple.getValues();
-
+
Assert.assertEquals(numberOfAttributes, values.size());
- for(int i = 0; i < numberOfAttributes; ++i) {
+ for (int i = 0; i < numberOfAttributes; ++i) {
Assert.assertEquals(flinkTuple.getField(i), values.get(i));
}
-
+
Assert.assertEquals(numberOfAttributes, tuple.size());
}
-
+
@Test
public void testBinary() {
final byte[] data = new byte[this.r.nextInt(15)];
this.r.nextBytes(data);
-
+
final int index = this.r.nextInt(5);
final Tuple flinkTuple = new Tuple5<Object, Object, Object, Object, Object>();
flinkTuple.setField(data, index);
-
+
final StormTuple<Tuple> tuple = new StormTuple<Tuple>(flinkTuple);
Assert.assertEquals(flinkTuple.getField(index), tuple.getBinary(index));
}
-
+
@Test
public void testBoolean() {
- final Boolean flinkTuple = new Boolean(this.r.nextBoolean());
-
+ final Boolean flinkTuple = this.r.nextBoolean();
+
final StormTuple<Boolean> tuple = new StormTuple<Boolean>(flinkTuple);
Assert.assertEquals(flinkTuple, tuple.getBoolean(0));
}
-
+
@Test
public void testByte() {
- final Byte flinkTuple = new Byte((byte)this.r.nextInt());
-
+ final Byte flinkTuple = (byte) this.r.nextInt();
+
final StormTuple<Byte> tuple = new StormTuple<Byte>(flinkTuple);
Assert.assertEquals(flinkTuple, tuple.getByte(0));
}
-
+
@Test
public void testDouble() {
- final Double flinkTuple = new Double(this.r.nextDouble());
-
+ final Double flinkTuple = this.r.nextDouble();
+
final StormTuple<Double> tuple = new StormTuple<Double>(flinkTuple);
Assert.assertEquals(flinkTuple, tuple.getDouble(0));
}
-
+
@Test
public void testFloat() {
- final Float flinkTuple = new Float(this.r.nextFloat());
-
+ final Float flinkTuple = this.r.nextFloat();
+
final StormTuple<Float> tuple = new StormTuple<Float>(flinkTuple);
Assert.assertEquals(flinkTuple, tuple.getFloat(0));
}
-
+
@Test
public void testInteger() {
- final Integer flinkTuple = new Integer(this.r.nextInt());
-
+ final Integer flinkTuple = this.r.nextInt();
+
final StormTuple<Integer> tuple = new StormTuple<Integer>(flinkTuple);
Assert.assertEquals(flinkTuple, tuple.getInteger(0));
}
-
+
@Test
public void testLong() {
- final Long flinkTuple = new Long(this.r.nextInt());
-
+ final Long flinkTuple = this.r.nextLong();
+
final StormTuple<Long> tuple = new StormTuple<Long>(flinkTuple);
Assert.assertEquals(flinkTuple, tuple.getLong(0));
}
-
+
@Test
public void testShort() {
- final Short flinkTuple = new Short((short)this.r.nextInt());
-
+ final Short flinkTuple = (short) this.r.nextInt();
+
final StormTuple<Short> tuple = new StormTuple<Short>(flinkTuple);
Assert.assertEquals(flinkTuple, tuple.getShort(0));
}
-
+
@Test
public void testString() {
final byte[] data = new byte[this.r.nextInt(15)];
this.r.nextBytes(data);
final String flinkTuple = new String(data);
-
+
final StormTuple<String> tuple = new StormTuple<String>(flinkTuple);
Assert.assertEquals(flinkTuple, tuple.getString(0));
}
-
+
@Test
public void testBinaryTuple() {
final byte[] data = new byte[this.r.nextInt(15)];
this.r.nextBytes(data);
-
+
final int index = this.r.nextInt(5);
final Tuple flinkTuple = new Tuple5<Object, Object, Object, Object, Object>();
flinkTuple.setField(data, index);
-
+
final StormTuple<Tuple> tuple = new StormTuple<Tuple>(flinkTuple);
Assert.assertEquals(flinkTuple.getField(index), tuple.getBinary(index));
}
-
+
@Test
public void testBooleanTuple() {
- final Boolean data = new Boolean(this.r.nextBoolean());
-
+ final Boolean data = this.r.nextBoolean();
+
final int index = this.r.nextInt(5);
final Tuple flinkTuple = new Tuple5<Object, Object, Object, Object, Object>();
flinkTuple.setField(data, index);
-
+
final StormTuple<Tuple> tuple = new StormTuple<Tuple>(flinkTuple);
Assert.assertEquals(flinkTuple.getField(index), tuple.getBoolean(index));
}
-
+
@Test
public void testByteTuple() {
- final Byte data = new Byte((byte)this.r.nextInt());
-
+ final Byte data = (byte) this.r.nextInt();
+
final int index = this.r.nextInt(5);
final Tuple flinkTuple = new Tuple5<Object, Object, Object, Object, Object>();
flinkTuple.setField(data, index);
-
+
final StormTuple<Tuple> tuple = new StormTuple<Tuple>(flinkTuple);
Assert.assertEquals(flinkTuple.getField(index), tuple.getByte(index));
}
-
+
@Test
public void testDoubleTuple() {
- final Double data = new Double(this.r.nextDouble());
-
+ final Double data = this.r.nextDouble();
+
final int index = this.r.nextInt(5);
final Tuple flinkTuple = new Tuple5<Object, Object, Object, Object, Object>();
flinkTuple.setField(data, index);
-
+
final StormTuple<Tuple> tuple = new StormTuple<Tuple>(flinkTuple);
Assert.assertEquals(flinkTuple.getField(index), tuple.getDouble(index));
}
-
+
@Test
public void testFloatTuple() {
- final Float data = new Float(this.r.nextFloat());
-
+ final Float data = this.r.nextFloat();
+
final int index = this.r.nextInt(5);
final Tuple flinkTuple = new Tuple5<Object, Object, Object, Object, Object>();
flinkTuple.setField(data, index);
-
+
final StormTuple<Tuple> tuple = new StormTuple<Tuple>(flinkTuple);
Assert.assertEquals(flinkTuple.getField(index), tuple.getFloat(index));
}
-
+
@Test
public void testIntegerTuple() {
- final Integer data = new Integer(this.r.nextInt());
-
+ final Integer data = this.r.nextInt();
+
final int index = this.r.nextInt(5);
final Tuple flinkTuple = new Tuple5<Object, Object, Object, Object, Object>();
flinkTuple.setField(data, index);
-
+
final StormTuple<Tuple> tuple = new StormTuple<Tuple>(flinkTuple);
Assert.assertEquals(flinkTuple.getField(index), tuple.getInteger(index));
}
-
+
@Test
public void testLongTuple() {
- final Long data = new Long(this.r.nextInt());
-
+ final Long data = this.r.nextLong();
+
final int index = this.r.nextInt(5);
final Tuple flinkTuple = new Tuple5<Object, Object, Object, Object, Object>();
flinkTuple.setField(data, index);
-
+
final StormTuple<Tuple> tuple = new StormTuple<Tuple>(flinkTuple);
Assert.assertEquals(flinkTuple.getField(index), tuple.getLong(index));
}
-
+
@Test
public void testShortTuple() {
- final Short data = new Short((short)this.r.nextInt());
-
+ final Short data = (short) this.r.nextInt();
+
final int index = this.r.nextInt(5);
final Tuple flinkTuple = new Tuple5<Object, Object, Object, Object, Object>();
flinkTuple.setField(data, index);
-
+
final StormTuple<Tuple> tuple = new StormTuple<Tuple>(flinkTuple);
Assert.assertEquals(flinkTuple.getField(index), tuple.getShort(index));
}
-
+
@Test
public void testStringTuple() {
final byte[] rawdata = new byte[this.r.nextInt(15)];
this.r.nextBytes(rawdata);
final String data = new String(rawdata);
-
+
final int index = this.r.nextInt(5);
final Tuple flinkTuple = new Tuple5<Object, Object, Object, Object, Object>();
flinkTuple.setField(data, index);
-
+
final StormTuple<Tuple> tuple = new StormTuple<Tuple>(flinkTuple);
Assert.assertEquals(flinkTuple.getField(index), tuple.getString(index));
}
-
+
@Test(expected = UnsupportedOperationException.class)
public void testContains() {
new StormTuple<Object>(null).contains(null);
}
-
+
@Test(expected = UnsupportedOperationException.class)
public void testGetFields() {
new StormTuple<Object>(null).getFields();
}
-
+
@Test(expected = UnsupportedOperationException.class)
public void testFieldIndex() {
new StormTuple<Object>(null).fieldIndex(null);
}
-
+
@Test(expected = UnsupportedOperationException.class)
public void testSelect() {
new StormTuple<Object>(null).select(null);
}
-
+
@Test(expected = UnsupportedOperationException.class)
public void testGetValueByField() {
new StormTuple<Object>(null).getValueByField(null);
}
-
+
@Test(expected = UnsupportedOperationException.class)
public void testGetStringByField() {
new StormTuple<Object>(null).getStringByField(null);
}
-
+
@Test(expected = UnsupportedOperationException.class)
public void testGetIntegerByField() {
new StormTuple<Object>(null).getIntegerByField(null);
}
-
+
@Test(expected = UnsupportedOperationException.class)
public void testGetLongByField() {
new StormTuple<Object>(null).getLongByField(null);
}
-
+
@Test(expected = UnsupportedOperationException.class)
public void testGetBooleanByField() {
new StormTuple<Object>(null).getBooleanByField(null);
}
-
+
@Test(expected = UnsupportedOperationException.class)
public void testGetShortByField() {
new StormTuple<Object>(null).getShortByField(null);
}
-
+
@Test(expected = UnsupportedOperationException.class)
public void testGetByteByField() {
new StormTuple<Object>(null).getByteByField(null);
}
-
+
@Test(expected = UnsupportedOperationException.class)
public void testGetDoubleByField() {
new StormTuple<Object>(null).getDoubleByField(null);
}
-
+
@Test(expected = UnsupportedOperationException.class)
public void testGetFloatByField() {
new StormTuple<Object>(null).getFloatByField(null);
}
-
+
@Test(expected = UnsupportedOperationException.class)
public void testGetBinaryByField() {
new StormTuple<Object>(null).getBinaryByField(null);
}
-
+
@Test(expected = UnsupportedOperationException.class)
public void testGetSourceGlobalStreamid() {
new StormTuple<Object>(null).getSourceGlobalStreamid();
}
-
+
@Test(expected = UnsupportedOperationException.class)
public void testGetSourceComponent() {
new StormTuple<Object>(null).getSourceComponent();
}
-
+
@Test(expected = UnsupportedOperationException.class)
public void testGetSourceTask() {
new StormTuple<Object>(null).getSourceTask();
}
-
+
@Test(expected = UnsupportedOperationException.class)
public void testGetSourceStreamId() {
new StormTuple<Object>(null).getSourceStreamId();
}
-
+
@Test(expected = UnsupportedOperationException.class)
public void testGetMessageId() {
new StormTuple<Object>(null).getMessageId();
}
-
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-compatibility/src/test/java/org/apache/flink/stormcompatibility/wrappers/StormWrapperSetupHelperTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-compatibility/src/test/java/org/apache/flink/stormcompatibility/wrappers/StormWrapperSetupHelperTest.java b/flink-staging/flink-streaming/flink-storm-compatibility/src/test/java/org/apache/flink/stormcompatibility/wrappers/StormWrapperSetupHelperTest.java
index d1451bf..15129ce 100644
--- a/flink-staging/flink-streaming/flink-storm-compatibility/src/test/java/org/apache/flink/stormcompatibility/wrappers/StormWrapperSetupHelperTest.java
+++ b/flink-staging/flink-streaming/flink-storm-compatibility/src/test/java/org/apache/flink/stormcompatibility/wrappers/StormWrapperSetupHelperTest.java
@@ -17,8 +17,10 @@
package org.apache.flink.stormcompatibility.wrappers;
-import static org.mockito.Mockito.mock;
-
+import backtype.storm.topology.IComponent;
+import backtype.storm.topology.IRichBolt;
+import backtype.storm.topology.IRichSpout;
+import backtype.storm.tuple.Fields;
import org.apache.flink.stormcompatibility.util.AbstractTest;
import org.junit.Assert;
import org.junit.Test;
@@ -27,123 +29,108 @@ import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
-import backtype.storm.topology.IComponent;
-import backtype.storm.topology.IRichBolt;
-import backtype.storm.topology.IRichSpout;
-import backtype.storm.tuple.Fields;
-
-
-
-
+import static org.mockito.Mockito.mock;
@RunWith(PowerMockRunner.class)
@PrepareForTest(StormWrapperSetupHelper.class)
public class StormWrapperSetupHelperTest extends AbstractTest {
-
+
@Test(expected = IllegalArgumentException.class)
public void testZeroAttributesDeclarerBolt() throws Exception {
IComponent boltOrSpout;
-
-
- if(this.r.nextBoolean()) {
+
+ if (this.r.nextBoolean()) {
boltOrSpout = mock(IRichSpout.class);
} else {
boltOrSpout = mock(IRichBolt.class);
}
-
+
final StormOutputFieldsDeclarer declarer = new StormOutputFieldsDeclarer();
declarer.declare(new Fields());
PowerMockito.whenNew(StormOutputFieldsDeclarer.class).withNoArguments().thenReturn(declarer);
-
+
StormWrapperSetupHelper.getNumberOfAttributes(boltOrSpout, this.r.nextBoolean());
}
-
+
@Test
public void testEmptyDeclarerBolt() {
IComponent boltOrSpout;
-
- if(this.r.nextBoolean()) {
+
+ if (this.r.nextBoolean()) {
boltOrSpout = mock(IRichSpout.class);
} else {
boltOrSpout = mock(IRichBolt.class);
}
-
+
Assert.assertEquals(-1, StormWrapperSetupHelper.getNumberOfAttributes(boltOrSpout, this.r.nextBoolean()));
}
-
-
+
@Test(expected = IllegalArgumentException.class)
public void testRawType() throws Exception {
IComponent boltOrSpout;
-
- if(this.r.nextBoolean()) {
+
+ if (this.r.nextBoolean()) {
boltOrSpout = mock(IRichSpout.class);
} else {
boltOrSpout = mock(IRichBolt.class);
}
-
+
final StormOutputFieldsDeclarer declarer = new StormOutputFieldsDeclarer();
declarer.declare(new Fields("dummy1", "dummy2"));
PowerMockito.whenNew(StormOutputFieldsDeclarer.class).withNoArguments().thenReturn(declarer);
-
+
StormWrapperSetupHelper.getNumberOfAttributes(boltOrSpout, true);
}
-
+
@Test(expected = IllegalArgumentException.class)
public void testToManyAttributes() throws Exception {
IComponent boltOrSpout;
-
- if(this.r.nextBoolean()) {
+
+ if (this.r.nextBoolean()) {
boltOrSpout = mock(IRichSpout.class);
} else {
boltOrSpout = mock(IRichBolt.class);
}
-
+
final StormOutputFieldsDeclarer declarer = new StormOutputFieldsDeclarer();
final String[] schema = new String[26];
- for(int i = 0; i < schema.length; ++i) {
+ for (int i = 0; i < schema.length; ++i) {
schema[i] = "a" + i;
}
declarer.declare(new Fields(schema));
PowerMockito.whenNew(StormOutputFieldsDeclarer.class).withNoArguments().thenReturn(declarer);
-
+
StormWrapperSetupHelper.getNumberOfAttributes(boltOrSpout, false);
}
-
-
+
@Test
public void testTupleTypes() throws Exception {
- for(int i = 0; i < 26; ++i) {
+ for (int i = 0; i < 26; ++i) {
this.testTupleTypes(i);
}
}
-
+
private void testTupleTypes(final int numberOfAttributes) throws Exception {
String[] schema = new String[numberOfAttributes];
- if(numberOfAttributes == 0) {
+ if (numberOfAttributes == 0) {
schema = new String[1];
}
- for(int i = 0; i < schema.length; ++i) {
+ for (int i = 0; i < schema.length; ++i) {
schema[i] = "a" + i;
}
-
+
IComponent boltOrSpout;
- if(this.r.nextBoolean()) {
+ if (this.r.nextBoolean()) {
boltOrSpout = mock(IRichSpout.class);
} else {
boltOrSpout = mock(IRichBolt.class);
}
-
+
final StormOutputFieldsDeclarer declarer = new StormOutputFieldsDeclarer();
declarer.declare(new Fields(schema));
PowerMockito.whenNew(StormOutputFieldsDeclarer.class).withNoArguments().thenReturn(declarer);
-
+
StormWrapperSetupHelper.getNumberOfAttributes(boltOrSpout, numberOfAttributes == 0);
}
-
- // @Test
- // public void testConvertToTopologyContext() {
- // Assert.fail();
- // }
-
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-compatibility/src/test/java/org/apache/flink/stormcompatibility/wrappers/TestCollector.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-compatibility/src/test/java/org/apache/flink/stormcompatibility/wrappers/TestCollector.java b/flink-staging/flink-streaming/flink-storm-compatibility/src/test/java/org/apache/flink/stormcompatibility/wrappers/TestCollector.java
index fc2768d..2d85597 100644
--- a/flink-staging/flink-streaming/flink-storm-compatibility/src/test/java/org/apache/flink/stormcompatibility/wrappers/TestCollector.java
+++ b/flink-staging/flink-streaming/flink-storm-compatibility/src/test/java/org/apache/flink/stormcompatibility/wrappers/TestCollector.java
@@ -17,26 +17,23 @@
package org.apache.flink.stormcompatibility.wrappers;
-import java.util.LinkedList;
-
import org.apache.flink.api.java.tuple.Tuple1;
import org.apache.flink.util.Collector;
-
-
-
+import java.util.LinkedList;
class TestCollector implements Collector<Tuple1<Integer>> {
public LinkedList<Tuple1<Integer>> result = new LinkedList<Tuple1<Integer>>();
-
- public TestCollector() {}
-
+
+ public TestCollector() {
+ }
+
@Override
public void collect(final Tuple1<Integer> record) {
this.result.add(record.copy());
}
-
+
@Override
public void close() {/* nothing to to */}
-
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-examples/src/assembly/word-count-storm.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-examples/src/assembly/word-count-storm.xml b/flink-staging/flink-streaming/flink-storm-examples/src/assembly/word-count-storm.xml
index 002d9bc..9721115 100644
--- a/flink-staging/flink-streaming/flink-storm-examples/src/assembly/word-count-storm.xml
+++ b/flink-staging/flink-streaming/flink-storm-examples/src/assembly/word-count-storm.xml
@@ -17,52 +17,56 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3
+ http://maven.apache.org/xsd/assembly-1.1.3.xsd">
- <id>WordCountStorm</id>
- <formats>
- <format>jar</format>
- </formats>
+ <id>WordCountStorm</id>
+ <formats>
+ <format>jar</format>
+ </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
+ <includeBaseDirectory>false</includeBaseDirectory>
- <dependencySets>
- <dependencySet>
- <outputDirectory>/</outputDirectory>
- <unpack>true</unpack>
- <includes>
- <!-- need to be added explicitly to get 'defaults.yaml' -->
- <include>org.apache.storm:storm-core:jar</include>
- <include>org.apache.flink:flink-storm-examples:jar</include>
- </includes>
- <unpackOptions>
- <includes>
- <!-- from storm-core -->
- <include>defaults.yaml</include>
- <include>backtype/storm/*.class</include>
- <include>backtype/storm/topology/*.class</include>
- <include>backtype/storm/spout/*.class</include>
- <include>backtype/storm/task/*.class</include>
- <include>backtype/storm/tuple/*.class</include>
- <include>backtype/storm/generated/*.class</include>
- <include>backtype/storm/metric/**/*.class</include>
- <include>backtype/storm/utils/*.class</include>
- <include>backtype/storm/serialization/*.class</include>
- <include>org/apache/storm/curator/**/*.class</include>
- <include>org/apache/thrift7/**/*.class</include>
- <!-- Storm's recursive dependencies -->
- <include>org/json/simple/**/*.class</include>
- <include>org/yaml/snakeyaml/**/*.class</include>
- <!-- compatibility layer -->
- <include>org/apache/flink/stormcompatibility/api/*.class</include>
- <include>org/apache/flink/stormcompatibility/wrappers/*.class</include>
- <!-- Word Count -->
- <include>org/apache/flink/stormcompatibility/wordcount/StormWordCountRemoteBySubmitter.class</include>
- <include>org/apache/flink/stormcompatibility/wordcount/WordCountTopology.class</include>
- <include>org/apache/flink/stormcompatibility/wordcount/stormoperators/*.class</include>
- <include>org/apache/flink/examples/java/wordcount/util/WordCountData.class</include>
- </includes>
- </unpackOptions>
- </dependencySet>
- </dependencySets>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>/</outputDirectory>
+ <unpack>true</unpack>
+ <includes>
+ <!-- need to be added explicitly to get 'defaults.yaml' -->
+ <include>org.apache.storm:storm-core:jar</include>
+ <include>org.apache.flink:flink-storm-examples:jar</include>
+ </includes>
+ <unpackOptions>
+ <includes>
+ <!-- from storm-core -->
+ <include>defaults.yaml</include>
+ <include>backtype/storm/*.class</include>
+ <include>backtype/storm/topology/*.class</include>
+ <include>backtype/storm/spout/*.class</include>
+ <include>backtype/storm/task/*.class</include>
+ <include>backtype/storm/tuple/*.class</include>
+ <include>backtype/storm/generated/*.class</include>
+ <include>backtype/storm/metric/**/*.class</include>
+ <include>backtype/storm/utils/*.class</include>
+ <include>backtype/storm/serialization/*.class</include>
+ <include>org/apache/storm/curator/**/*.class</include>
+ <include>org/apache/thrift7/**/*.class</include>
+ <!-- Storm's recursive dependencies -->
+ <include>org/json/simple/**/*.class</include>
+ <include>org/yaml/snakeyaml/**/*.class</include>
+ <!-- compatibility layer -->
+ <include>org/apache/flink/stormcompatibility/api/*.class</include>
+ <include>org/apache/flink/stormcompatibility/wrappers/*.class</include>
+ <!-- Word Count -->
+ <include>org/apache/flink/stormcompatibility/wordcount/StormWordCountRemoteBySubmitter.class
+ </include>
+ <include>org/apache/flink/stormcompatibility/wordcount/WordCountTopology.class</include>
+ <include>org/apache/flink/stormcompatibility/wordcount/stormoperators/*.class</include>
+ <include>org/apache/flink/examples/java/wordcount/util/WordCountData.class</include>
+ </includes>
+ </unpackOptions>
+ </dependencySet>
+ </dependencySets>
</assembly>
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/util/AbstractStormBoltSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/util/AbstractStormBoltSink.java b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/util/AbstractStormBoltSink.java
new file mode 100644
index 0000000..939380c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/util/AbstractStormBoltSink.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.stormcompatibility.util;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.IRichBolt;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Tuple;
+
+import java.util.Map;
+
+/**
+ * Implements a sink that write the received data so some external output. The result is formatted like
+ * {@code (a1, a2, ..., an)} with {@code Object.toString()} for each attribute).
+ */
+public abstract class AbstractStormBoltSink implements IRichBolt {
+ private static final long serialVersionUID = -1626323806848080430L;
+
+ private StringBuilder lineBuilder;
+ private String prefix = "";
+ private OutputFormatter formatter;
+
+ public AbstractStormBoltSink(OutputFormatter formatter) {
+ this.formatter = formatter;
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public final void prepare(final Map stormConf, final TopologyContext context,
+ final OutputCollector collector) {
+ this.prepareSimple(stormConf, context);
+ if (context.getComponentCommon(context.getThisComponentId()).get_parallelism_hint() > 1) {
+ this.prefix = context.getThisTaskId() + "> ";
+ }
+ }
+
+ protected abstract void prepareSimple(final Map<?, ?> stormConf, final TopologyContext context);
+
+ @Override
+ public final void execute(final Tuple input) {
+ this.lineBuilder = new StringBuilder();
+ this.lineBuilder.append(this.prefix);
+ lineBuilder.append(formatter.format(input));
+// this.lineBuilder.append("(");
+// for (final Object attribute : input.getValues()) {
+// this.lineBuilder.append(attribute);
+// this.lineBuilder.append(",");
+// }
+// this.lineBuilder.replace(this.lineBuilder.length() - 1, this.lineBuilder.length(), ")");
+
+ this.writeExternal(this.lineBuilder.toString());
+ }
+
+ protected abstract void writeExternal(final String line);
+
+ @Override
+ public void cleanup() {/* nothing to do */}
+
+ @Override
+ public final void declareOutputFields(final OutputFieldsDeclarer declarer) {/* nothing to do */}
+
+ @Override
+ public Map<String, Object> getComponentConfiguration() {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/util/AbstractStormSpout.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/util/AbstractStormSpout.java b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/util/AbstractStormSpout.java
new file mode 100644
index 0000000..d4d185f
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/util/AbstractStormSpout.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.stormcompatibility.util;
+
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.IRichSpout;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
+
+import java.util.Map;
+
+/**
+ * Base class for Storm Spout that reads data line by line from an arbitrary source. The declared output schema has a
+ * single attribute calle {@code line} and should be of type {@link String}.
+ */
+public abstract class AbstractStormSpout implements IRichSpout {
+ private static final long serialVersionUID = 8876828403487806771L;
+
+ public final static String ATTRIBUTE_LINE = "line";
+
+ protected SpoutOutputCollector collector;
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public void open(final Map conf, final TopologyContext context, final SpoutOutputCollector collector) {
+ this.collector = collector;
+ }
+
+ @Override
+ public void close() {/* noting to do */}
+
+ @Override
+ public void activate() {/* noting to do */}
+
+ @Override
+ public void deactivate() {/* noting to do */}
+
+ @Override
+ public void ack(final Object msgId) {/* noting to do */}
+
+ @Override
+ public void fail(final Object msgId) {/* noting to do */}
+
+ @Override
+ public void declareOutputFields(final OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields(ATTRIBUTE_LINE));
+ }
+
+ @Override
+ public Map<String, Object> getComponentConfiguration() {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/util/OutputFormatter.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/util/OutputFormatter.java b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/util/OutputFormatter.java
new file mode 100644
index 0000000..0de7b5c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/util/OutputFormatter.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.stormcompatibility.util;
+
+public interface OutputFormatter {
+
+ public String format(Object input);
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/util/StormBoltFileSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/util/StormBoltFileSink.java b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/util/StormBoltFileSink.java
new file mode 100644
index 0000000..86b55aa
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/util/StormBoltFileSink.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.stormcompatibility.util;
+
+import backtype.storm.task.TopologyContext;
+
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Implements a sink that write the received data to the given file (as a result of {@code Object.toString()} for each
+ * attribute).
+ */
+public final class StormBoltFileSink extends AbstractStormBoltSink {
+ private static final long serialVersionUID = 2014027288631273666L;
+
+ private final String path;
+ private BufferedWriter writer;
+
+ public StormBoltFileSink(final String path, OutputFormatter formatter) {
+ super(formatter);
+ this.path = path;
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public void prepareSimple(final Map stormConf, final TopologyContext context) {
+ try {
+ this.writer = new BufferedWriter(new FileWriter(this.path));
+ } catch (final IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void writeExternal(final String line) {
+ try {
+ this.writer.write(line + "\n");
+ } catch (final IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void cleanup() {
+ if (this.writer != null) {
+ try {
+ this.writer.close();
+ } catch (final IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/util/StormBoltPrintSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/util/StormBoltPrintSink.java b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/util/StormBoltPrintSink.java
new file mode 100644
index 0000000..3bf49d0
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/util/StormBoltPrintSink.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.stormcompatibility.util;
+
+import backtype.storm.task.TopologyContext;
+
+import java.util.Map;
+
+/**
+ * Implements a sink that prints the received data to {@code stdout}.
+ */
+public final class StormBoltPrintSink extends AbstractStormBoltSink {
+ private static final long serialVersionUID = -6650011223001009519L;
+
+ public StormBoltPrintSink(OutputFormatter formatter) {
+ super(formatter);
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public void prepareSimple(final Map stormConf, final TopologyContext context) {
+ /* nothing to do */
+ }
+
+ @Override
+ public void writeExternal(final String line) {
+ System.out.println(line);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/BoltTokenizerWordCount.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/BoltTokenizerWordCount.java b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/BoltTokenizerWordCount.java
index 8a7fc4f..ab8d027 100644
--- a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/BoltTokenizerWordCount.java
+++ b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/BoltTokenizerWordCount.java
@@ -17,6 +17,7 @@
package org.apache.flink.stormcompatibility.wordcount;
+import backtype.storm.topology.IRichBolt;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.examples.java.wordcount.util.WordCountData;
@@ -25,82 +26,76 @@ import org.apache.flink.stormcompatibility.wrappers.StormBoltWrapper;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import backtype.storm.topology.IRichBolt;
-
-
-
-
-
/**
* Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming
* fashion. The tokenizer step is performed by a Storm {@link IRichBolt bolt}.
- *
- * <p>
+ * <p/>
+ * <p/>
* The input is a plain text file with lines separated by newline characters.
- *
- * <p>
+ * <p/>
+ * <p/>
* Usage: <code>WordCount <text path> <result path></code><br>
* If no parameters are provided, the program is run with default data from {@link WordCountData}.
- *
- * <p>
+ * <p/>
+ * <p/>
* This example shows how to:
* <ul>
* <li>use a Storm bolt within a Flink Streaming program.
* </ul>
*/
public class BoltTokenizerWordCount {
-
+
// *************************************************************************
// PROGRAM
// *************************************************************************
-
+
public static void main(final String[] args) throws Exception {
-
- if(!parseParameters(args)) {
+
+ if (!parseParameters(args)) {
return;
}
-
+
// set up the execution environment
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
+
// get input data
final DataStream<String> text = getTextDataStream(env);
-
+
final DataStream<Tuple2<String, Integer>> counts = text
- // split up the lines in pairs (2-tuples) containing: (word,1)
- // this is done by a Storm bolt that is wrapped accordingly
- .transform("StormBoltTokenizer",
- TypeExtractor.getForObject(new Tuple2<String, Integer>(new String(), new Integer(0))),
- new StormBoltWrapper<String, Tuple2<String, Integer>>(new StormBoltTokenizer()))
- // split up the lines in pairs (2-tuples) containing: (word,1)
- // group by the tuple field "0" and sum up tuple field "1"
- .groupBy(0).sum(1);
-
+ // split up the lines in pairs (2-tuples) containing: (word,1)
+ // this is done by a Storm bolt that is wrapped accordingly
+ .transform("StormBoltTokenizer",
+ TypeExtractor.getForObject(new Tuple2<String, Integer>("", 0)),
+ new StormBoltWrapper<String, Tuple2<String, Integer>>(new StormBoltTokenizer()))
+ // split up the lines in pairs (2-tuples) containing: (word,1)
+ // group by the tuple field "0" and sum up tuple field "1"
+ .groupBy(0).sum(1);
+
// emit result
- if(fileOutput) {
+ if (fileOutput) {
counts.writeAsText(outputPath);
} else {
counts.print();
}
-
+
// execute program
env.execute("Streaming WordCount with Storm bolt tokenizer");
}
-
+
// *************************************************************************
// UTIL METHODS
// *************************************************************************
-
+
private static boolean fileOutput = false;
private static String textPath;
private static String outputPath;
-
+
private static boolean parseParameters(final String[] args) {
-
- if(args.length > 0) {
+
+ if (args.length > 0) {
// parse input arguments
fileOutput = true;
- if(args.length == 2) {
+ if (args.length == 2) {
textPath = args[0];
outputPath = args[1];
} else {
@@ -108,20 +103,20 @@ public class BoltTokenizerWordCount {
return false;
}
} else {
- System.out.println("Executing WordCount example with built-in default data.");
- System.out.println(" Provide parameters to read input data from a file.");
+ System.out.println("Executing WordCount example with built-in default data");
+ System.out.println(" Provide parameters to read input data from a file");
System.out.println(" Usage: WordCount <text path> <result path>");
}
return true;
}
-
+
private static DataStream<String> getTextDataStream(final StreamExecutionEnvironment env) {
- if(fileOutput) {
+ if (fileOutput) {
// read the text file from given input path
return env.readTextFile(textPath);
}
-
+
return env.fromElements(WordCountData.WORDS);
}
-
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/SpoutSourceWordCount.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/SpoutSourceWordCount.java b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/SpoutSourceWordCount.java
index caec8eb..4178efa 100644
--- a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/SpoutSourceWordCount.java
+++ b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/SpoutSourceWordCount.java
@@ -17,6 +17,7 @@
package org.apache.flink.stormcompatibility.wordcount;
+import backtype.storm.topology.IRichSpout;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.TypeExtractor;
@@ -28,103 +29,97 @@ import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;
-import backtype.storm.topology.IRichSpout;
-
-
-
-
-
/**
* Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming
* fashion. The used data source is a Storm {@link IRichSpout bolt}.
- *
- * <p>
+ * <p/>
+ * <p/>
* The input is a plain text file with lines separated by newline characters.
- *
- * <p>
+ * <p/>
+ * <p/>
* Usage: <code>WordCount <text path> <result path></code><br>
* If no parameters are provided, the program is run with default data from {@link WordCountData}.
- *
- * <p>
+ * <p/>
+ * <p/>
* This example shows how to:
* <ul>
* <li>use a Storm bolt within a Flink Streaming program.
* </ul>
*/
public class SpoutSourceWordCount {
-
+
// *************************************************************************
// PROGRAM
// *************************************************************************
-
+
public static void main(final String[] args) throws Exception {
-
- if(!parseParameters(args)) {
+
+ if (!parseParameters(args)) {
return;
}
-
+
// set up the execution environment
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
+
// get input data
final DataStream<String> text = getTextDataStream(env);
-
+
final DataStream<Tuple2<String, Integer>> counts =
- // split up the lines in pairs (2-tuples) containing: (word,1)
- text.flatMap(new Tokenizer())
- // group by the tuple field "0" and sum up tuple field "1"
- .groupBy(0).sum(1);
-
+ // split up the lines in pairs (2-tuples) containing: (word,1)
+ text.flatMap(new Tokenizer())
+ // group by the tuple field "0" and sum up tuple field "1"
+ .groupBy(0).sum(1);
+
// emit result
- if(fileOutput) {
+ if (fileOutput) {
counts.writeAsText(outputPath);
} else {
counts.print();
}
-
+
// execute program
env.execute("Streaming WordCount with Storm spout source");
}
-
+
// *************************************************************************
// USER FUNCTIONS
// *************************************************************************
-
+
/**
* Implements the string tokenizer that splits sentences into words as a user-defined FlatMapFunction. The function
* takes a line (String) and splits it into multiple pairs in the form of "(word,1)" (Tuple2<String, Integer>).
*/
public static final class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {
private static final long serialVersionUID = 1L;
-
+
@Override
public void flatMap(final String value, final Collector<Tuple2<String, Integer>> out) throws Exception {
// normalize and split the line
final String[] tokens = value.toLowerCase().split("\\W+");
-
+
// emit the pairs
- for(final String token : tokens) {
- if(token.length() > 0) {
- out.collect(new Tuple2<String, Integer>(token, new Integer(1)));
+ for (final String token : tokens) {
+ if (token.length() > 0) {
+ out.collect(new Tuple2<String, Integer>(token, 1));
}
}
}
}
-
+
// *************************************************************************
// UTIL METHODS
// *************************************************************************
-
+
private static boolean fileOutput = false;
private static String textPath;
private static String outputPath;
-
+
private static boolean parseParameters(final String[] args) {
-
- if(args.length > 0) {
+
+ if (args.length > 0) {
// parse input arguments
fileOutput = true;
- if(args.length == 2) {
+ if (args.length == 2) {
textPath = args[0];
outputPath = args[1];
} else {
@@ -132,27 +127,26 @@ public class SpoutSourceWordCount {
return false;
}
} else {
- System.out.println("Executing WordCount example with built-in default data.");
- System.out.println(" Provide parameters to read input data from a file.");
+ System.out.println("Executing WordCount example with built-in default data");
+ System.out.println(" Provide parameters to read input data from a file");
System.out.println(" Usage: WordCount <text path> <result path>");
}
return true;
}
-
+
private static DataStream<String> getTextDataStream(final StreamExecutionEnvironment env) {
- if(fileOutput) {
+ if (fileOutput) {
// read the text file from given input path
final String[] tokens = textPath.split(":");
final String localFile = tokens[tokens.length - 1];
- final DataStream<String> stream = env.addSource(
- new StormFiniteSpoutWrapper<String>(new StormFileSpout(localFile), true),
- TypeExtractor.getForClass(String.class)).setParallelism(1);
- return stream;
+ return env.addSource(
+ new StormFiniteSpoutWrapper<String>(new StormFileSpout(localFile), true),
+ TypeExtractor.getForClass(String.class)).setParallelism(1);
}
-
+
return env.addSource(new StormFiniteSpoutWrapper<String>(new StormInMemorySpout(), true),
- TypeExtractor.getForClass(String.class));
-
+ TypeExtractor.getForClass(String.class));
+
}
-
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/StormWordCountLocal.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/StormWordCountLocal.java b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/StormWordCountLocal.java
index 99a2c8e..7b4f471 100644
--- a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/StormWordCountLocal.java
+++ b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/StormWordCountLocal.java
@@ -17,34 +17,29 @@
package org.apache.flink.stormcompatibility.wordcount;
-import org.apache.flink.examples.java.wordcount.util.WordCountData;
-import org.apache.flink.stormcompatibility.api.FlinkLocalCluster;
-import org.apache.flink.stormcompatibility.api.FlinkTopologyBuilder;
-
import backtype.storm.LocalCluster;
import backtype.storm.generated.StormTopology;
import backtype.storm.utils.Utils;
-
-
-
-
+import org.apache.flink.examples.java.wordcount.util.WordCountData;
+import org.apache.flink.stormcompatibility.api.FlinkLocalCluster;
+import org.apache.flink.stormcompatibility.api.FlinkTopologyBuilder;
/**
* Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming
* fashion. The program is constructed as a regular {@link StormTopology} and submitted to Flink for execution in the
* same way as to a Storm {@link LocalCluster}.
- *
+ * <p/>
* This example shows how to run program directly within Java, thus it cannot be used to submit a {@link StormTopology}
* via Flink command line clients (ie, bin/flink).
- *
- * <p>
+ * <p/>
+ * <p/>
* The input is a plain text file with lines separated by newline characters.
- *
- * <p>
+ * <p/>
+ * <p/>
* Usage: <code>WordCount <text path> <result path></code><br>
* If no parameters are provided, the program is run with default data from {@link WordCountData}.
- *
- * <p>
+ * <p/>
+ * <p/>
* This example shows how to:
* <ul>
* <li>run a regular Storm program locally on Flink
@@ -52,29 +47,29 @@ import backtype.storm.utils.Utils;
*/
public class StormWordCountLocal {
public final static String topologyId = "Streaming WordCount";
-
+
// *************************************************************************
// PROGRAM
// *************************************************************************
-
+
public static void main(final String[] args) throws Exception {
-
- if(!WordCountTopology.parseParameters(args)) {
+
+ if (!WordCountTopology.parseParameters(args)) {
return;
}
-
+
// build Topology the Storm way
final FlinkTopologyBuilder builder = WordCountTopology.buildTopology();
-
+
// execute program locally
final FlinkLocalCluster cluster = FlinkLocalCluster.getLocalCluster();
cluster.submitTopology(topologyId, null, builder.createTopology());
-
+
Utils.sleep(5 * 1000);
-
+
// TODO kill does no do anything so far
cluster.killTopology(topologyId);
cluster.shutdown();
}
-
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/StormWordCountRemoteByClient.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/StormWordCountRemoteByClient.java b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/StormWordCountRemoteByClient.java
index 7f3a496..9e56c14 100644
--- a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/StormWordCountRemoteByClient.java
+++ b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/StormWordCountRemoteByClient.java
@@ -17,10 +17,6 @@
package org.apache.flink.stormcompatibility.wordcount;
-import org.apache.flink.examples.java.wordcount.util.WordCountData;
-import org.apache.flink.stormcompatibility.api.FlinkClient;
-import org.apache.flink.stormcompatibility.api.FlinkTopologyBuilder;
-
import backtype.storm.Config;
import backtype.storm.generated.AlreadyAliveException;
import backtype.storm.generated.InvalidTopologyException;
@@ -28,27 +24,26 @@ import backtype.storm.generated.NotAliveException;
import backtype.storm.generated.StormTopology;
import backtype.storm.utils.NimbusClient;
import backtype.storm.utils.Utils;
-
-
-
-
+import org.apache.flink.examples.java.wordcount.util.WordCountData;
+import org.apache.flink.stormcompatibility.api.FlinkClient;
+import org.apache.flink.stormcompatibility.api.FlinkTopologyBuilder;
/**
* Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming
* fashion. The program is constructed as a regular {@link StormTopology} and submitted to Flink for execution in the
* same way as to a Storm cluster similar to {@link NimbusClient}. The Flink cluster can be local or remote.
- *
+ * <p/>
* This example shows how to submit the program via Java, thus it cannot be used to submit a {@link StormTopology} via
* Flink command line clients (ie, bin/flink).
- *
- * <p>
+ * <p/>
+ * <p/>
* The input is a plain text file with lines separated by newline characters.
- *
- * <p>
+ * <p/>
+ * <p/>
* Usage: <code>WordCount <text path> <result path></code><br>
* If no parameters are provided, the program is run with default data from {@link WordCountData}.
- *
- * <p>
+ * <p/>
+ * <p/>
* This example shows how to:
* <ul>
* <li>submit a regular Storm program to a local or remote Flink cluster.
@@ -57,32 +52,34 @@ import backtype.storm.utils.Utils;
public class StormWordCountRemoteByClient {
public final static String topologyId = "Streaming WordCount";
private final static String uploadedJarLocation = "target/flink-storm-examples-0.9-SNAPSHOT-WordCountStorm.jar";
-
+
// *************************************************************************
// PROGRAM
// *************************************************************************
-
+
public static void main(final String[] args) throws AlreadyAliveException, InvalidTopologyException,
- NotAliveException {
-
- if(!WordCountTopology.parseParameters(args)) {
+ NotAliveException {
+
+ if (!WordCountTopology.parseParameters(args)) {
return;
}
-
+
// build Topology the Storm way
final FlinkTopologyBuilder builder = WordCountTopology.buildTopology();
-
+
// execute program on Flink cluster
final Config conf = new Config();
- conf.put(Config.NIMBUS_HOST, "localhost"); // can be changed to remote address
- conf.put(Config.NIMBUS_THRIFT_PORT, new Integer(6123)); // use default flink jobmanger.rpc.port
-
+ // can be changed to remote address
+ conf.put(Config.NIMBUS_HOST, "localhost");
+ // use default flink jobmanger.rpc.port
+ conf.put(Config.NIMBUS_THRIFT_PORT, 6123);
+
final FlinkClient cluster = FlinkClient.getConfiguredClient(conf);
- cluster.submitTopology(topologyId, uploadedJarLocation, null, builder.createTopology());
-
+ cluster.submitTopology(topologyId, uploadedJarLocation, builder.createTopology());
+
Utils.sleep(5 * 1000);
-
+
cluster.killTopology(topologyId);
}
-
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/StormWordCountRemoteBySubmitter.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/StormWordCountRemoteBySubmitter.java b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/StormWordCountRemoteBySubmitter.java
index dadca67..a1fb79d 100644
--- a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/StormWordCountRemoteBySubmitter.java
+++ b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/StormWordCountRemoteBySubmitter.java
@@ -17,34 +17,29 @@
package org.apache.flink.stormcompatibility.wordcount;
+import backtype.storm.Config;
+import backtype.storm.StormSubmitter;
+import backtype.storm.generated.StormTopology;
import org.apache.flink.examples.java.wordcount.util.WordCountData;
import org.apache.flink.stormcompatibility.api.FlinkClient;
import org.apache.flink.stormcompatibility.api.FlinkSubmitter;
import org.apache.flink.stormcompatibility.api.FlinkTopologyBuilder;
-import backtype.storm.Config;
-import backtype.storm.StormSubmitter;
-import backtype.storm.generated.StormTopology;
-
-
-
-
-
/**
* Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming
* fashion. The program is constructed as a regular {@link StormTopology} and submitted to Flink for execution in the
* same way as to a Storm cluster similar to {@link StormSubmitter}. The Flink cluster can be local or remote.
- *
+ * <p/>
* This example shows how to submit the program via Java as well as Flink's command line client (ie, bin/flink).
- *
- * <p>
+ * <p/>
+ * <p/>
* The input is a plain text file with lines separated by newline characters.
- *
- * <p>
+ * <p/>
+ * <p/>
* Usage: <code>WordCount <text path> <result path></code><br>
* If no parameters are provided, the program is run with default data from {@link WordCountData}.
- *
- * <p>
+ * <p/>
+ * <p/>
* This example shows how to:
* <ul>
* <li>submit a regular Storm program to a local or remote Flink cluster.
@@ -52,38 +47,37 @@ import backtype.storm.generated.StormTopology;
*/
public class StormWordCountRemoteBySubmitter {
public final static String topologyId = "Streaming WordCount";
-
+
// *************************************************************************
// PROGRAM
// *************************************************************************
-
+
public static void main(final String[] args) throws Exception {
-
- if(!WordCountTopology.parseParameters(args)) {
+
+ if (!WordCountTopology.parseParameters(args)) {
return;
}
-
+
// build Topology the Storm way
final FlinkTopologyBuilder builder = WordCountTopology.buildTopology();
-
+
// execute program on Flink cluster
-
final Config conf = new Config();
- // we can set Jobmanager host/port values manually or leave them blank
+ // We can set Jobmanager host/port values manually or leave them blank
// if not set and
// - executed within Java, default values "localhost" and "6123" are set by FlinkSubmitter
- // - executed via bin/flink values from flink-conf.yaml are set by FlinkSubmitter
- // conf.put(Config.NIMBUS_HOST, "localhost"); // can be changed to remote address
- // conf.put(Config.NIMBUS_THRIFT_PORT, new Integer(6123)); // use default flink jobmanger.rpc.port
-
- // the user jar file must be specified via JVM argument if executed via Java
+ // - executed via bin/flink values from flink-conf.yaml are set by FlinkSubmitter.
+ // conf.put(Config.NIMBUS_HOST, "localhost");
+ // conf.put(Config.NIMBUS_THRIFT_PORT, new Integer(6123));
+
+ // The user jar file must be specified via JVM argument if executed via Java.
// => -Dstorm.jar=target/flink-storm-examples-0.9-SNAPSHOT-WordCountStorm.jar
- // if bin/flink is used, the jar file is detected automatically
+ // If bin/flink is used, the jar file is detected automatically.
FlinkSubmitter.submitTopology(topologyId, conf, builder.createTopology());
-
+
Thread.sleep(5 * 1000);
-
+
FlinkClient.getConfiguredClient(conf).killTopology(topologyId);
}
-
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/WordCountTopology.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/WordCountTopology.java b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/WordCountTopology.java
index c9bf7b9..975b039 100644
--- a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/WordCountTopology.java
+++ b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/WordCountTopology.java
@@ -17,34 +17,31 @@
package org.apache.flink.stormcompatibility.wordcount;
+import backtype.storm.generated.StormTopology;
+import backtype.storm.tuple.Fields;
import org.apache.flink.examples.java.wordcount.util.WordCountData;
import org.apache.flink.stormcompatibility.api.FlinkTopologyBuilder;
+import org.apache.flink.stormcompatibility.util.OutputFormatter;
+import org.apache.flink.stormcompatibility.util.StormBoltFileSink;
+import org.apache.flink.stormcompatibility.util.StormBoltPrintSink;
import org.apache.flink.stormcompatibility.wordcount.stormoperators.StormBoltCounter;
-import org.apache.flink.stormcompatibility.wordcount.stormoperators.StormBoltFileSink;
-import org.apache.flink.stormcompatibility.wordcount.stormoperators.StormBoltPrintSink;
import org.apache.flink.stormcompatibility.wordcount.stormoperators.StormBoltTokenizer;
import org.apache.flink.stormcompatibility.wordcount.stormoperators.StormFileSpout;
import org.apache.flink.stormcompatibility.wordcount.stormoperators.StormInMemorySpout;
-
-import backtype.storm.generated.StormTopology;
-import backtype.storm.tuple.Fields;
-
-
-
-
+import org.apache.flink.stormcompatibility.wordcount.stormoperators.WordCountOutputFormatter;
/**
* Implements the "WordCount" program that computes a simple word occurrence histogram over text files in a streaming
* fashion. The program is constructed as a regular {@link StormTopology}.
- *
- * <p>
+ * <p/>
+ * <p/>
* The input is a plain text file with lines separated by newline characters.
- *
- * <p>
+ * <p/>
+ * <p/>
* Usage: <code>WordCount <text path> <result path></code><br>
* If no parameters are provided, the program is run with default data from {@link WordCountData}.
- *
- * <p>
+ * <p/>
+ * <p/>
* This example shows how to:
* <ul>
* <li>how to construct a regular Storm topology as Flink program
@@ -55,15 +52,14 @@ public class WordCountTopology {
public final static String tokenierzerId = "tokenizer";
public final static String counterId = "counter";
public final static String sinkId = "sink";
-
-
-
+ private final static OutputFormatter formatter = new WordCountOutputFormatter();
+
public static FlinkTopologyBuilder buildTopology() {
-
+
final FlinkTopologyBuilder builder = new FlinkTopologyBuilder();
-
+
// get input data
- if(fileInputOutput) {
+ if (fileInputOutput) {
// read the text file from given input path
final String[] tokens = textPath.split(":");
final String inputFile = tokens[tokens.length - 1];
@@ -71,40 +67,40 @@ public class WordCountTopology {
} else {
builder.setSpout(spoutId, new StormInMemorySpout());
}
-
+
// split up the lines in pairs (2-tuples) containing: (word,1)
- builder.setBolt(tokenierzerId, new StormBoltTokenizer(), new Integer(4)).shuffleGrouping(spoutId);
+ builder.setBolt(tokenierzerId, new StormBoltTokenizer(), 4).shuffleGrouping(spoutId);
// group by the tuple field "0" and sum up tuple field "1"
- builder.setBolt(counterId, new StormBoltCounter(), new Integer(4)).fieldsGrouping(tokenierzerId,
- new Fields(StormBoltTokenizer.ATTRIBUTE_WORD));
-
+ builder.setBolt(counterId, new StormBoltCounter(), 4).fieldsGrouping(tokenierzerId,
+ new Fields(StormBoltTokenizer.ATTRIBUTE_WORD));
+
// emit result
- if(fileInputOutput) {
+ if (fileInputOutput) {
// read the text file from given input path
final String[] tokens = outputPath.split(":");
final String outputFile = tokens[tokens.length - 1];
- builder.setBolt(sinkId, new StormBoltFileSink(outputFile)).shuffleGrouping(counterId);
+ builder.setBolt(sinkId, new StormBoltFileSink(outputFile, formatter)).shuffleGrouping(counterId);
} else {
- builder.setBolt(sinkId, new StormBoltPrintSink(), new Integer(4)).shuffleGrouping(counterId);
+ builder.setBolt(sinkId, new StormBoltPrintSink(formatter), 4).shuffleGrouping(counterId);
}
-
+
return builder;
}
-
+
// *************************************************************************
// UTIL METHODS
// *************************************************************************
-
+
private static boolean fileInputOutput = false;
private static String textPath;
private static String outputPath;
-
+
static boolean parseParameters(final String[] args) {
-
- if(args.length > 0) {
+
+ if (args.length > 0) {
// parse input arguments
fileInputOutput = true;
- if(args.length == 2) {
+ if (args.length == 2) {
textPath = args[0];
outputPath = args[1];
} else {
@@ -112,12 +108,12 @@ public class WordCountTopology {
return false;
}
} else {
- System.out.println("Executing WordCount example with built-in default data.");
- System.out.println(" Provide parameters to read input data from a file.");
+ System.out.println("Executing WordCount example with built-in default data");
+ System.out.println(" Provide parameters to read input data from a file");
System.out.println(" Usage: WordCount <text path> <result path>");
}
-
+
return true;
}
-
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/AbstractStormBoltSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/AbstractStormBoltSink.java b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/AbstractStormBoltSink.java
deleted file mode 100644
index ee21f7e..0000000
--- a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/AbstractStormBoltSink.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.stormcompatibility.wordcount.stormoperators;
-
-import java.util.Map;
-
-import backtype.storm.task.OutputCollector;
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.IRichBolt;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.tuple.Tuple;
-
-
-
-
-
-/**
- * Implements a sink that write the received data so some external output. The result is formatted like
- * {@code (a1,a2,...,an)} with {@code Object.toString()} for each attribute).
- */
-public abstract class AbstractStormBoltSink implements IRichBolt {
- private static final long serialVersionUID = -1626323806848080430L;
-
- private StringBuilder lineBuilder;
- private String prefix = "";
-
-
-
- @Override
- public final void prepare(@SuppressWarnings("rawtypes") final Map stormConf, final TopologyContext context, final OutputCollector collector) {
- this.prepareSimple(stormConf, context);
- if(context.getComponentCommon(context.getThisComponentId()).get_parallelism_hint() > 1) {
- this.prefix = context.getThisTaskId() + "> ";
- }
- }
-
- protected abstract void prepareSimple(final Map<?, ?> stormConf, final TopologyContext context);
-
- @Override
- public final void execute(final Tuple input) {
- this.lineBuilder = new StringBuilder();
- this.lineBuilder.append(this.prefix);
- this.lineBuilder.append("(");
- for(final Object attribute : input.getValues()) {
- this.lineBuilder.append(attribute);
- this.lineBuilder.append(",");
- }
- this.lineBuilder.replace(this.lineBuilder.length() - 1, this.lineBuilder.length(), ")");
-
- this.writeExternal(this.lineBuilder.toString());
- }
-
- protected abstract void writeExternal(final String line);
-
- @Override
- public void cleanup() {/* nothing to do */}
-
- @Override
- public final void declareOutputFields(final OutputFieldsDeclarer declarer) {/* nothing to do */}
-
- @Override
- public Map<String, Object> getComponentConfiguration() {
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/AbstractStormSpout.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/AbstractStormSpout.java b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/AbstractStormSpout.java
deleted file mode 100644
index 3639b3c..0000000
--- a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/AbstractStormSpout.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.stormcompatibility.wordcount.stormoperators;
-
-import java.util.Map;
-
-import backtype.storm.spout.SpoutOutputCollector;
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.IRichSpout;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.tuple.Fields;
-
-
-
-
-
-/**
- * Base class for Storm Spout that reads data line by line from an arbitrary source. The declared output schema has a
- * single attribute calle {@code line} and should be of type {@link String}.
- */
-public abstract class AbstractStormSpout implements IRichSpout {
- private static final long serialVersionUID = 8876828403487806771L;
-
- public final static String ATTRIBUTE_LINE = "line";
- public final static int ATTRIBUTE_LINE_INDEX = 0;
-
- protected SpoutOutputCollector collector;
-
-
-
- @Override
- public void open(@SuppressWarnings("rawtypes") final Map conf, final TopologyContext context, @SuppressWarnings("hiding") final SpoutOutputCollector collector) {
- this.collector = collector;
- }
-
- @Override
- public void close() {/* noting to do */}
-
- @Override
- public void activate() {/* noting to do */}
-
- @Override
- public void deactivate() {/* noting to do */}
-
- @Override
- public void ack(final Object msgId) {/* noting to do */}
-
- @Override
- public void fail(final Object msgId) {/* noting to do */}
-
- @Override
- public void declareOutputFields(final OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields(ATTRIBUTE_LINE));
- }
-
- @Override
- public Map<String, Object> getComponentConfiguration() {
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/StormBoltCounter.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/StormBoltCounter.java b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/StormBoltCounter.java
index 8b9777b..f1122ab 100644
--- a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/StormBoltCounter.java
+++ b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/StormBoltCounter.java
@@ -14,10 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.flink.stormcompatibility.wordcount.stormoperators;
-import java.util.HashMap;
-import java.util.Map;
+package org.apache.flink.stormcompatibility.wordcount.stormoperators;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
@@ -27,9 +25,8 @@ import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
-
-
-
+import java.util.HashMap;
+import java.util.Map;
/**
* Implements the word counter that the occurrence of each unique word. The bolt takes a pair (input tuple schema:
@@ -38,59 +35,55 @@ import backtype.storm.tuple.Values;
*/
public class StormBoltCounter implements IRichBolt {
private static final long serialVersionUID = 399619605462625934L;
-
+
public static final String ATTRIBUTE_WORD = "word";
public static final String ATTRIBUTE_COUNT = "count";
-
- public static final int ATTRIBUTE_WORD_INDEX = 0;
- public static final int ATTRIBUTE_COUNT_INDEX = 1;
-
+
private final HashMap<String, Count> counts = new HashMap<String, Count>();
private OutputCollector collector;
-
-
-
+
+ @SuppressWarnings("rawtypes")
@Override
- public void prepare(@SuppressWarnings("rawtypes") final Map stormConf, final TopologyContext context, @SuppressWarnings("hiding") final OutputCollector collector) {
+ public void prepare(final Map stormConf, final TopologyContext context, final OutputCollector collector) {
this.collector = collector;
}
-
+
@Override
public void execute(final Tuple input) {
final String word = input.getString(StormBoltTokenizer.ATTRIBUTE_WORD_INDEX);
-
+
Count currentCount = this.counts.get(word);
- if(currentCount == null) {
+ if (currentCount == null) {
currentCount = new Count();
this.counts.put(word, currentCount);
}
- currentCount.count += input.getInteger(StormBoltTokenizer.ATTRIBUTE_COUNT_INDEX).intValue();
-
- this.collector.emit(new Values(word, new Integer(currentCount.count)));
+ currentCount.count += input.getInteger(StormBoltTokenizer.ATTRIBUTE_COUNT_INDEX);
+
+ this.collector.emit(new Values(word, currentCount.count));
}
-
+
@Override
public void cleanup() {/* nothing to do */}
-
+
@Override
public void declareOutputFields(final OutputFieldsDeclarer declarer) {
declarer.declare(new Fields(ATTRIBUTE_WORD, ATTRIBUTE_COUNT));
}
-
+
@Override
public Map<String, Object> getComponentConfiguration() {
return null;
}
-
+
/**
* A counter helper to emit immutable tuples to the given collector and avoid unnecessary object creating/deletion.
- *
+ *
* @author mjsax
*/
private static final class Count {
public int count;
-
+
public Count() {/* nothing to do */}
}
-
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/StormBoltFileSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/StormBoltFileSink.java b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/StormBoltFileSink.java
deleted file mode 100644
index 2be8c5d..0000000
--- a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/StormBoltFileSink.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.stormcompatibility.wordcount.stormoperators;
-
-import java.io.BufferedWriter;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.Map;
-
-import backtype.storm.task.TopologyContext;
-
-
-
-
-
-/**
- * Implements a sink that write the received data to the given file (as a result of {@code Object.toString()} for each
- * attribute).
- */
-public final class StormBoltFileSink extends AbstractStormBoltSink {
- private static final long serialVersionUID = 2014027288631273666L;
-
- private final String path;
- private BufferedWriter writer;
-
-
-
- public StormBoltFileSink(final String path) {
- this.path = path;
- }
-
-
-
- @Override
- public void prepareSimple(@SuppressWarnings("rawtypes") final Map stormConf, final TopologyContext context) {
- try {
- this.writer = new BufferedWriter(new FileWriter(this.path));
- } catch(final IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void writeExternal(final String line) {
- try {
- this.writer.write(line + "\n");
- } catch(final IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void cleanup() {
- if(this.writer != null) {
- try {
- this.writer.close();
- } catch(final IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/StormBoltPrintSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/StormBoltPrintSink.java b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/StormBoltPrintSink.java
deleted file mode 100644
index 2d61c99..0000000
--- a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/StormBoltPrintSink.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.stormcompatibility.wordcount.stormoperators;
-
-import java.util.Map;
-
-import backtype.storm.task.TopologyContext;
-
-
-
-
-
-/**
- * Implements a sink that prints the received data to {@code stdout}.
- */
-public final class StormBoltPrintSink extends AbstractStormBoltSink {
- private static final long serialVersionUID = -6650011223001009519L;
-
- @Override
- public void prepareSimple(@SuppressWarnings("rawtypes") final Map stormConf, final TopologyContext context) {
- /* nothing to do */
- }
-
- @Override
- public void writeExternal(final String line) {
- System.out.println(line);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/e497a831/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/StormBoltTokenizer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/StormBoltTokenizer.java b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/StormBoltTokenizer.java
index 6e30665..96bd87c 100644
--- a/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/StormBoltTokenizer.java
+++ b/flink-staging/flink-streaming/flink-storm-examples/src/main/java/org/apache/flink/stormcompatibility/wordcount/stormoperators/StormBoltTokenizer.java
@@ -14,9 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.flink.stormcompatibility.wordcount.stormoperators;
-import java.util.Map;
+package org.apache.flink.stormcompatibility.wordcount.stormoperators;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
@@ -26,9 +25,7 @@ import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
-
-
-
+import java.util.Map;
/**
* Implements the string tokenizer that splits sentences into words as a Storm bolt. The bolt takes a line (input tuple
@@ -37,44 +34,43 @@ import backtype.storm.tuple.Values;
*/
public final class StormBoltTokenizer implements IRichBolt {
private static final long serialVersionUID = -8589620297208175149L;
-
+
public static final String ATTRIBUTE_WORD = "word";
public static final String ATTRIBUTE_COUNT = "count";
-
+
public static final int ATTRIBUTE_WORD_INDEX = 0;
public static final int ATTRIBUTE_COUNT_INDEX = 1;
-
+
private OutputCollector collector;
-
-
-
+
+ @SuppressWarnings("rawtypes")
@Override
- public void prepare(@SuppressWarnings("rawtypes") final Map stormConf, final TopologyContext context, @SuppressWarnings("hiding") final OutputCollector collector) {
+ public void prepare(final Map stormConf, final TopologyContext context, final OutputCollector collector) {
this.collector = collector;
}
-
+
@Override
public void execute(final Tuple input) {
final String[] tokens = input.getString(0).toLowerCase().split("\\W+");
-
- for(final String token : tokens) {
- if(token.length() > 0) {
- this.collector.emit(new Values(token, new Integer(1)));
+
+ for (final String token : tokens) {
+ if (token.length() > 0) {
+ this.collector.emit(new Values(token, 1));
}
}
}
-
+
@Override
public void cleanup() {/* nothing to do */}
-
+
@Override
public void declareOutputFields(final OutputFieldsDeclarer declarer) {
declarer.declare(new Fields(ATTRIBUTE_WORD, ATTRIBUTE_COUNT));
}
-
+
@Override
public Map<String, Object> getComponentConfiguration() {
return null;
}
-
+
}