You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by pt...@apache.org on 2015/06/04 04:05:51 UTC
[09/50] [abbrv] storm git commit: link examples to source
link examples to source
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a791604a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a791604a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a791604a
Branch: refs/heads/0.10.x-branch
Commit: a791604ac5d274cededae8a01362dac4dfd82d84
Parents: 0c1e0aa
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Tue Apr 7 00:07:49 2015 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Tue Apr 7 00:07:49 2015 -0400
----------------------------------------------------------------------
.../src/test/resources/configs/hdfs_test.yaml | 97 ++++++++++++++++++++
flux-examples/README.md | 8 +-
2 files changed, 101 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/a791604a/flux-core/src/test/resources/configs/hdfs_test.yaml
----------------------------------------------------------------------
diff --git a/flux-core/src/test/resources/configs/hdfs_test.yaml b/flux-core/src/test/resources/configs/hdfs_test.yaml
new file mode 100644
index 0000000..8fe0a9a
--- /dev/null
+++ b/flux-core/src/test/resources/configs/hdfs_test.yaml
@@ -0,0 +1,97 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Test ability to wire together shell spouts/bolts
+---
+
+# topology definition
+# name to be used when submitting
+name: "hdfs-topology"
+
+# Components
+# Components are analagous to Spring beans. They are meant to be used as constructor,
+# property(setter), and builder arguments.
+#
+# for the time being, components must be declared in the order they are referenced
+components:
+ - id: "syncPolicy"
+ className: "org.apache.storm.hdfs.bolt.sync.CountSyncPolicy"
+ constructorArgs:
+ - 1000
+ - id: "rotationPolicy"
+ className: "org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy"
+ constructorArgs:
+ - 5.0
+ - MB
+
+ - id: "fileNameFormat"
+ className: "org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat"
+ configMethods:
+ - name: "withPath"
+ args: ["/tmp/foo/"]
+ - name: "withExtension"
+ args: [".txt"]
+
+ - id: "recordFormat"
+ className: "org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat"
+ configMethods:
+ - name: "withFieldDelimiter"
+ args: ["|"]
+
+ - id: "rotationAction"
+ className: "org.apache.storm.hdfs.common.rotation.MoveFileAction"
+ configMethods:
+ - name: "toDestination"
+ args: ["/tmp/dest2"]
+
+# spout definitions
+spouts:
+ - id: "spout-1"
+ className: "backtype.storm.testing.TestWordSpout"
+ parallelism: 1
+ # ...
+
+# bolt definitions
+
+# HdfsBolt bolt = new HdfsBolt()
+# .withConfigKey("hdfs.config")
+# .withFsUrl(args[0])
+# .withFileNameFormat(fileNameFormat)
+# .withRecordFormat(format)
+# .withRotationPolicy(rotationPolicy)
+# .withSyncPolicy(syncPolicy)
+# .addRotationAction(new MoveFileAction().toDestination("/tmp/dest2/"));
+bolts:
+ - id: "bolt-1"
+ className: "org.apache.storm.hdfs.bolt.HdfsBolt"
+ configMethods:
+ - name: "withConfigKey"
+ args: ["hdfs.config"]
+ - name: "withFsUrl"
+ args: ["hdfs://hadoop:54310"]
+ - name: "withFileNameFormat"
+ args: [ref: "fileNameFormat"]
+ - name: "withRecordFormat"
+ args: [ref: "recordFormat"]
+ - name: "withRotationPolicy"
+ args: [ref: "rotationPolicy"]
+ - name: "withSyncPolicy"
+ args: [ref: "syncPolicy"]
+ - name: "addRotationAction"
+ args: [ref: "rotationAction"]
+ parallelism: 1
+ # ...
+
http://git-wip-us.apache.org/repos/asf/storm/blob/a791604a/flux-examples/README.md
----------------------------------------------------------------------
diff --git a/flux-examples/README.md b/flux-examples/README.md
index 9f5682e..425ad98 100644
--- a/flux-examples/README.md
+++ b/flux-examples/README.md
@@ -28,21 +28,21 @@ storm jar ./target/flux-examples-0.2.3-SNAPSHOT.jar org.apache.storm.flux.Flux -
## Available Examples
-### simple_wordcount.yaml
+### [simple_wordcount.yaml](src/main/resources/simple_wordcount.yaml)
This is a very basic wordcount example using Java spouts and bolts. It simply logs the running count of each word
received.
-### multilang.yaml
+### [multilang.yaml](src/main/resources/multilang.yaml)
Another wordcount example that uses a spout written in JavaScript (node.js), a bolt written in Python, and two bolts
written in java.
-### kafka_spout.yaml
+### [kafka_spout.yaml](src/main/resources/kafka_spout.yaml)
This example illustrates how to configure Storm's `storm-kafka` spout using Flux YAML DSL `components`, `references`,
and `constructor arguments` constructs.
-### simple_hdfs.yaml
+### [simple_hdfs.yaml](src/main/resources/simple_hdfs.yaml)
This example demonstrates using Flux to setup a storm-hdfs bolt to write to an HDFS cluster. It also demonstrates Flux's
variable substitution/filtering feature.