You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by pt...@apache.org on 2015/06/04 04:05:56 UTC
[14/50] [abbrv] storm git commit: update hbase and hdfs examples
update hbase and hdfs examples
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/9f5f8227
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/9f5f8227
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/9f5f8227
Branch: refs/heads/0.10.x-branch
Commit: 9f5f822720544e356682d3c007c3407f5c2e5058
Parents: 01702dc
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Wed Apr 8 12:50:59 2015 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Wed Apr 8 12:50:59 2015 -0400
----------------------------------------------------------------------
.../storm/flux/examples/WordCountClient.java | 17 +++++++--
flux-examples/src/main/resources/hbase-site.xml | 36 --------------------
.../src/main/resources/hbase_bolt.properties | 2 ++
.../src/main/resources/hdfs_bolt.properties | 6 ++--
.../src/main/resources/simple_hbase.yaml | 3 +-
.../src/main/resources/simple_hdfs.yaml | 6 ++--
6 files changed, 24 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/9f5f8227/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java
----------------------------------------------------------------------
diff --git a/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java b/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java
index 55873d5..eb4fb7a 100644
--- a/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java
+++ b/flux-examples/src/main/java/org/apache/storm/flux/examples/WordCountClient.java
@@ -24,6 +24,9 @@ import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
+import java.io.FileInputStream;
+import java.util.Properties;
+
/**
* Connects to the 'WordCount' HBase table and prints counts for each word.
*
@@ -39,8 +42,17 @@ public class WordCountClient {
public static void main(String[] args) throws Exception {
Configuration config = HBaseConfiguration.create();
- if(args.length > 0){
- config.set("hbase.rootdir", args[0]);
+ if(args.length == 1){
+ Properties props = new Properties();
+ props.load(new FileInputStream(args[0]));
+ System.out.println("HBase configuration:");
+ for(Object key : props.keySet()) {
+ System.out.println(key + "=" + props.get(key));
+ config.set((String)key, props.getProperty((String)key));
+ }
+ } else {
+ System.out.println("Usage: WordCountClient <hbase_config.properties>");
+ System.exit(1);
}
HTable table = new HTable(config, "WordCount");
@@ -54,7 +66,6 @@ public class WordCountClient {
byte[] wordBytes = result.getValue(Bytes.toBytes("cf"), Bytes.toBytes("word"));
String wordStr = Bytes.toString(wordBytes);
- System.out.println(wordStr);
long count = Bytes.toLong(countBytes);
System.out.println("Word: '" + wordStr + "', Count: " + count);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/9f5f8227/flux-examples/src/main/resources/hbase-site.xml
----------------------------------------------------------------------
diff --git a/flux-examples/src/main/resources/hbase-site.xml b/flux-examples/src/main/resources/hbase-site.xml
deleted file mode 100644
index 06c3031..0000000
--- a/flux-examples/src/main/resources/hbase-site.xml
+++ /dev/null
@@ -1,36 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration>
- <property>
- <name>hbase.cluster.distributed</name>
- <value>true</value>
- </property>
- <property>
- <name>hbase.rootdir</name>
- <value>hdfs://hadoop:54310/hbase</value>
- </property>
- <property>
- <name>hbase.zookeeper.quorum</name>
- <value>hadoop</value>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/9f5f8227/flux-examples/src/main/resources/hbase_bolt.properties
----------------------------------------------------------------------
diff --git a/flux-examples/src/main/resources/hbase_bolt.properties b/flux-examples/src/main/resources/hbase_bolt.properties
new file mode 100644
index 0000000..9903b41
--- /dev/null
+++ b/flux-examples/src/main/resources/hbase_bolt.properties
@@ -0,0 +1,2 @@
+hbase.rootdir=hdfs://hadoop:54310/hbase
+hbase.zookeeper.quorum=hadoop
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/9f5f8227/flux-examples/src/main/resources/hdfs_bolt.properties
----------------------------------------------------------------------
diff --git a/flux-examples/src/main/resources/hdfs_bolt.properties b/flux-examples/src/main/resources/hdfs_bolt.properties
index 34a7a23..dd1307d 100644
--- a/flux-examples/src/main/resources/hdfs_bolt.properties
+++ b/flux-examples/src/main/resources/hdfs_bolt.properties
@@ -1,9 +1,9 @@
# The HDFS url
-hdfs.url="hdfs://hadoop:54310"
+hdfs.url=hdfs://hadoop:54310
# The HDFS directory where the bolt will write incoming data
-hdfs.write.dir="/incoming"
+hdfs.write.dir=/incoming
# The HDFS directory where files will be moved once the bolt has
# finished writing to it.
-hdfs.dest.dir="/complete"
\ No newline at end of file
+hdfs.dest.dir=/complete
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/9f5f8227/flux-examples/src/main/resources/simple_hbase.yaml
----------------------------------------------------------------------
diff --git a/flux-examples/src/main/resources/simple_hbase.yaml b/flux-examples/src/main/resources/simple_hbase.yaml
index 5eb70ed..62686d0 100644
--- a/flux-examples/src/main/resources/simple_hbase.yaml
+++ b/flux-examples/src/main/resources/simple_hbase.yaml
@@ -51,7 +51,8 @@ components:
config:
topology.workers: 1
hbase.conf:
- hbase.rootdir: "hdfs://hadoop:54310/hbase"
+ hbase.rootdir: "${hbase.rootdir}"
+ hbase.zookeeper.quorum: "${hbase.zookeeper.quorum}"
# spout definitions
spouts:
http://git-wip-us.apache.org/repos/asf/storm/blob/9f5f8227/flux-examples/src/main/resources/simple_hdfs.yaml
----------------------------------------------------------------------
diff --git a/flux-examples/src/main/resources/simple_hdfs.yaml b/flux-examples/src/main/resources/simple_hdfs.yaml
index ea7721d..9007869 100644
--- a/flux-examples/src/main/resources/simple_hdfs.yaml
+++ b/flux-examples/src/main/resources/simple_hdfs.yaml
@@ -41,7 +41,7 @@ components:
className: "org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat"
configMethods:
- name: "withPath"
- args: [${hdfs.write.dir}]
+ args: ["${hdfs.write.dir}"]
- name: "withExtension"
args: [".txt"]
@@ -55,7 +55,7 @@ components:
className: "org.apache.storm.hdfs.common.rotation.MoveFileAction"
configMethods:
- name: "toDestination"
- args: [${hdfs.dest.dir}]
+ args: ["${hdfs.dest.dir}"]
# spout definitions
spouts:
@@ -73,7 +73,7 @@ bolts:
- name: "withConfigKey"
args: ["hdfs.config"]
- name: "withFsUrl"
- args: [${hdfs.url}]
+ args: ["${hdfs.url}"]
- name: "withFileNameFormat"
args: [ref: "fileNameFormat"]
- name: "withRecordFormat"