You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by pt...@apache.org on 2015/11/05 21:41:35 UTC
[56/60] [abbrv] storm git commit: remove jstorm-utility directory
http://git-wip-us.apache.org/repos/asf/storm/blob/e8f64d5e/jstorm-utility/topology-monitor/src/main/java/com/dianping/cosmos/util/JSONUtil.java
----------------------------------------------------------------------
diff --git a/jstorm-utility/topology-monitor/src/main/java/com/dianping/cosmos/util/JSONUtil.java b/jstorm-utility/topology-monitor/src/main/java/com/dianping/cosmos/util/JSONUtil.java
deleted file mode 100755
index 673576c..0000000
--- a/jstorm-utility/topology-monitor/src/main/java/com/dianping/cosmos/util/JSONUtil.java
+++ /dev/null
@@ -1,125 +0,0 @@
-package com.dianping.cosmos.util;
-
-import java.io.IOException;
-import java.io.StringWriter;
-import java.util.List;
-import java.util.Map;
-
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonGenerator;
-import org.codehaus.jackson.JsonNode;
-import org.codehaus.jackson.JsonProcessingException;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * JSON工具类
- * @author xinchun.wang
- *
- */
-public class JSONUtil {
- private static final Logger LOGGER = LoggerFactory.getLogger(JSONUtil.class);
-
- private static final ObjectMapper MAPPER = new ObjectMapper();
-
- private static final JSONUtil INSTANCE = new JSONUtil();
-
- public static JSONUtil getInstance() {
- return INSTANCE;
- }
-
- private JSONUtil() {
-
- }
-
- /**
- * 将map转化为json
- *
- * @param map
- * @return
- */
- public String formatMap2JSON(Map<String, Object> map) {
- StringWriter stringWriter = new StringWriter();
- String json = "";
- try {
- JsonGenerator gen = new JsonFactory()
- .createJsonGenerator(stringWriter);
- MAPPER.writeValue(gen, map);
- gen.close();
- json = stringWriter.toString();
- } catch (Exception e) {
- LOGGER.error("", e);
- }
- return json;
- }
-
- /**
- * POJO对象转换为JSON
- * @param pojo
- * @return
- */
- public String formatPOJO2JSON(Object pojo) {
- StringWriter stringWriter = new StringWriter();
- String json = "";
- try {
- JsonGenerator gen = new JsonFactory()
- .createJsonGenerator(stringWriter);
- MAPPER.writeValue(gen, pojo);
- gen.close();
- json = stringWriter.toString();
- } catch (Exception e) {
- LOGGER.error(pojo.getClass().getName() + "转json出错", e);
- }
- return json;
- }
-
- /**
- * 将json转化为map
- *
- * @param json
- * @return
- */
- public Map<?, ?> formatJSON2Map(String json) {
- Map<?, ?> map = null;
- try {
- map = MAPPER.readValue(json, Map.class);
- } catch (Exception e) {
- LOGGER.error("formatJsonToMap error, json = " + json, e);
- }
- return map;
- }
-
-
-
- /**
- * JSON转换为List
- * @param json
- * @return
- */
- public List<?> formatJSON2List(String json) {
- List<?> list = null;
- try {
- list = MAPPER.readValue(json, List.class);
- } catch (Exception e) {
- LOGGER.error("formatJSON2List error, json = " + json, e);
- }
- return list;
- }
-
- public boolean equals(String firstJSON, String secondJSON) {
- try {
- JsonNode tree1 = MAPPER.readTree(firstJSON);
- JsonNode tree2 = MAPPER.readTree(secondJSON);
- boolean areEqual = tree1.equals(tree2);
- return areEqual;
- } catch (JsonProcessingException e) {
- LOGGER.error("json compare wrong:" + firstJSON + ";" + secondJSON,
- e);
- } catch (IOException e) {
- LOGGER.error("json compare wrong:" + firstJSON + ";" + secondJSON,
- e);
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/e8f64d5e/jstorm-utility/topology-monitor/src/main/java/com/dianping/cosmos/util/TupleHelpers.java
----------------------------------------------------------------------
diff --git a/jstorm-utility/topology-monitor/src/main/java/com/dianping/cosmos/util/TupleHelpers.java b/jstorm-utility/topology-monitor/src/main/java/com/dianping/cosmos/util/TupleHelpers.java
deleted file mode 100755
index ebcf5bc..0000000
--- a/jstorm-utility/topology-monitor/src/main/java/com/dianping/cosmos/util/TupleHelpers.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.dianping.cosmos.util;
-
-import backtype.storm.Constants;
-import backtype.storm.tuple.Tuple;
-
-public final class TupleHelpers {
-
- private TupleHelpers() {
- }
-
- public static boolean isTickTuple(Tuple tuple) {
- return tuple.getSourceComponent().equals(Constants.SYSTEM_COMPONENT_ID) && tuple.getSourceStreamId().equals(
- Constants.SYSTEM_TICK_STREAM_ID);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/e8f64d5e/jstorm-utility/transaction_meta_spout/conf/topology.yaml
----------------------------------------------------------------------
diff --git a/jstorm-utility/transaction_meta_spout/conf/topology.yaml b/jstorm-utility/transaction_meta_spout/conf/topology.yaml
deleted file mode 100644
index 8ebe492..0000000
--- a/jstorm-utility/transaction_meta_spout/conf/topology.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-
-meta.consumer.group: "S_BATCH_META_TEST"
-meta.topic: "jae_log"
-meta.subexpress: "*"
-meta.nameserver: null
-#type "yyyyMMddhhmmss"
-#meta.start.timestamp: "20140701000000"
-meta.start.timestamp: null
-meta.batch.message.num: 1024
-meta.max.fail.times: 10
-meta.rebalance.frequency.sec: 60
-
-
-storm.cluster.mode: "local"
-#storm.cluster.mode: "distributed"
-
-topology.name: "batch_test"
-topology.spout.parallel: 1
-topology.bolt.parallel: 1
-topology.message.timeout.secs: 300
-topology.workers: 10
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/e8f64d5e/jstorm-utility/transaction_meta_spout/pom.xml
----------------------------------------------------------------------
diff --git a/jstorm-utility/transaction_meta_spout/pom.xml b/jstorm-utility/transaction_meta_spout/pom.xml
deleted file mode 100644
index fd0e1c7..0000000
--- a/jstorm-utility/transaction_meta_spout/pom.xml
+++ /dev/null
@@ -1,68 +0,0 @@
-<?xml version="1.0"?>
-<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>com.alibaba.aloha</groupId>
- <artifactId>aloha-utility</artifactId>
- <version>0.0.1-SNAPSHOT</version>
- </parent>
- <artifactId>transaction_meta_spout</artifactId>
-
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <descriptorRefs>
- <descriptorRef>jar-with-dependencies</descriptorRef>
- </descriptorRefs>
- </configuration>
- <executions>
- <execution>
- <id>make-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>1.6</source>
- <target>1.6</target>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
- <dependencies>
- <dependency>
- <groupId>com.taobao.metaq.final</groupId>
- <artifactId>metaq-client</artifactId>
- <version>3.1.3</version>
- </dependency>
- <!--
- <dependency>
- <groupId>com.alibaba.rocketmq</groupId>
- <artifactId>rocketmq-common</artifactId>
- <version>3.0.1</version>
- </dependency>
- <dependency>
- <groupId>com.alibaba.rocketmq</groupId>
- <artifactId>rocketmq-client</artifactId>
- <version>3.0.1</version>
- </dependency>
- <dependency>
- <groupId>com.alibaba.rocketmq</groupId>
- <artifactId>rocketmq-remoting</artifactId>
- <version>3.0.1</version>
- </dependency>
- -->
-
- </dependencies>
-
-</project>
http://git-wip-us.apache.org/repos/asf/storm/blob/e8f64d5e/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/BatchMetaRebalance.java
----------------------------------------------------------------------
diff --git a/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/BatchMetaRebalance.java b/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/BatchMetaRebalance.java
deleted file mode 100644
index bdab899..0000000
--- a/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/BatchMetaRebalance.java
+++ /dev/null
@@ -1,108 +0,0 @@
-package com.alibaba.jstorm.batch.example;
-
-import java.util.Map;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.log4j.Logger;
-
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.BasicOutputCollector;
-import backtype.storm.topology.IBasicBolt;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.tuple.Fields;
-import backtype.storm.tuple.Tuple;
-import backtype.storm.tuple.Values;
-
-import com.alibaba.jstorm.batch.BatchId;
-import com.alibaba.jstorm.batch.IPostCommit;
-import com.alibaba.jstorm.utils.JStormUtils;
-
-public class BatchMetaRebalance implements IBasicBolt, IPostCommit {
- /** */
- private static final long serialVersionUID = 2991323223385556163L;
-
- private static final Logger LOG = Logger
- .getLogger(BatchMetaRebalance.class);
-
- public static final String BOLT_NAME = BatchMetaRebalance.class
- .getSimpleName();
-
- public static final String REBALANCE_STREAM_ID = "rebalance";
-
- private transient AtomicBoolean isNeedRebalance;
- private transient ScheduledExecutorService scheduExec;
-
- @Override
- public void prepare(Map stormConf, TopologyContext context) {
- isNeedRebalance = new AtomicBoolean(false);
- CheckRebalanceTimer timer = new CheckRebalanceTimer();
-
- int rebalanceTimeInterval = JStormUtils.parseInt(
- stormConf.get("meta.rebalance.frequency.sec"), 3600);
-
- long now = System.currentTimeMillis();
- long next = (now/(rebalanceTimeInterval * 1000) + 1) * rebalanceTimeInterval * 1000;
- long diff = (next - now )/1000;
-
-// Calendar start = Calendar.getInstance();
-//
-// start.add(Calendar.HOUR_OF_DAY, 1);
-//
-// start.set(Calendar.MINUTE, 30);
-// start.set(Calendar.SECOND, 0);
-// start.set(Calendar.MILLISECOND, 0);
-// long startMillis = start.getTimeInMillis();
-//
-// long now = System.currentTimeMillis();
-//
-// long diff = (startMillis - now) / (1000);
-
- ScheduledExecutorService scheduExec = null;
-
- scheduExec = Executors.newSingleThreadScheduledExecutor();
- scheduExec.scheduleAtFixedRate(timer, diff, rebalanceTimeInterval,
- TimeUnit.SECONDS);
-
- LOG.info("Successfully init rebalance timer");
- }
-
- @Override
- public void execute(Tuple input, BasicOutputCollector collector) {
- LOG.warn("Receive unkonw message");
- }
-
- @Override
- public void cleanup() {
- scheduExec.shutdown();
- LOG.info("Successfully do cleanup");
- }
-
- @Override
- public void postCommit(BatchId id, BasicOutputCollector collector) {
- if (isNeedRebalance.get() == true) {
- isNeedRebalance.set(false);
- collector.emit(REBALANCE_STREAM_ID, new Values(id));
- LOG.info("Emit rebalance command");
- }
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declareStream(REBALANCE_STREAM_ID, new Fields("BatchId"));
- }
-
- @Override
- public Map<String, Object> getComponentConfiguration() {
- return null;
- }
-
- public class CheckRebalanceTimer implements Runnable {
- public void run() {
- BatchMetaRebalance.this.isNeedRebalance.set(true);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/e8f64d5e/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/BatchMetaSpout.java
----------------------------------------------------------------------
diff --git a/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/BatchMetaSpout.java b/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/BatchMetaSpout.java
deleted file mode 100644
index 8effd5b..0000000
--- a/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/BatchMetaSpout.java
+++ /dev/null
@@ -1,131 +0,0 @@
-package com.alibaba.jstorm.batch.example;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.log4j.Logger;
-
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.BasicOutputCollector;
-import backtype.storm.topology.FailedException;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.tuple.Fields;
-import backtype.storm.tuple.Tuple;
-import backtype.storm.tuple.Values;
-
-import com.alibaba.jstorm.batch.BatchId;
-import com.alibaba.jstorm.batch.IBatchSpout;
-import com.alibaba.jstorm.batch.meta.MetaSimpleClient;
-import com.alibaba.jstorm.batch.meta.MetaSpoutConfig;
-import com.alibaba.jstorm.batch.util.BatchCommon;
-import com.alibaba.jstorm.cluster.ClusterState;
-import com.alibaba.rocketmq.common.message.MessageExt;
-
-public class BatchMetaSpout implements IBatchSpout{
- private static final long serialVersionUID = 5720810158625748041L;
-
- private static final Logger LOG = Logger.getLogger(BatchMetaSpout.class);
-
- public static final String SPOUT_NAME = BatchMetaSpout.class.getSimpleName();
-
- private Map conf;
-
- private String taskName;
- private int taskIndex;
- private int taskParallel;
-
- private transient MetaSimpleClient metaClient;
- private MetaSpoutConfig metaSpoutConfig;
-
-
-
- public BatchMetaSpout(MetaSpoutConfig metaSpoutConfig) {
- this.metaSpoutConfig = metaSpoutConfig;
- }
-
- public void initMetaClient() throws Exception {
- ClusterState zkClient = BatchCommon.getZkClient(conf);
- metaClient = new MetaSimpleClient(metaSpoutConfig, zkClient, taskIndex,
- taskParallel);
-
- metaClient.init();
-
- LOG.info("Successfully init meta client " + taskName);
- }
-
- @Override
- public void prepare(Map stormConf, TopologyContext context) {
- this.conf = stormConf;
-
- taskName = context.getThisComponentId() + "_" + context.getThisTaskId();
-
- taskIndex = context.getThisTaskIndex();
-
- taskParallel = context.getComponentTasks(context.getThisComponentId())
- .size();
-
- try {
- initMetaClient();
- } catch (Exception e) {
- LOG.info("Failed to init Meta Client,", e);
- throw new RuntimeException(e);
- }
-
- LOG.info(taskName + " successfully do prepare ");
- }
-
- public void emitBatch(BatchId batchId, BasicOutputCollector collector) {
- List<MessageExt> msgs = metaClient.fetchOneBatch();
- for (MessageExt msgExt : msgs) {
- collector.emit(new Values(batchId, msgExt));
- }
- }
-
- @Override
- public void execute(Tuple input, BasicOutputCollector collector) {
-
- String streamId = input.getSourceStreamId();
- if (streamId.equals(BatchMetaRebalance.REBALANCE_STREAM_ID)) {
- try {
- metaClient.rebalanceMqList();
- } catch (Exception e) {
- LOG.warn("Failed to do rebalance operation", e);
- }
- }else {
- BatchId batchId = (BatchId) input.getValue(0);
-
- emitBatch(batchId, collector);
- }
-
- }
-
- @Override
- public void cleanup() {
- metaClient.cleanup();
-
- }
-
- @Override
- public byte[] commit(BatchId id) throws FailedException {
- return metaClient.commit(id);
- }
-
- @Override
- public void revert(BatchId id, byte[] commitResult) {
- metaClient.revert(id, commitResult);
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("BatchId", "MessageExt"));
- }
-
- @Override
- public Map<String, Object> getComponentConfiguration() {
- return null;
- }
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/e8f64d5e/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/BatchMetaTopology.java
----------------------------------------------------------------------
diff --git a/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/BatchMetaTopology.java b/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/BatchMetaTopology.java
deleted file mode 100644
index 63c017d..0000000
--- a/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/BatchMetaTopology.java
+++ /dev/null
@@ -1,163 +0,0 @@
-package com.alibaba.jstorm.batch.example;
-
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.InputStream;
-import java.util.Date;
-import java.util.Map;
-
-import org.yaml.snakeyaml.Yaml;
-
-import backtype.storm.Config;
-import backtype.storm.StormSubmitter;
-import backtype.storm.generated.AlreadyAliveException;
-import backtype.storm.generated.InvalidTopologyException;
-import backtype.storm.generated.TopologyAssignException;
-import backtype.storm.topology.BoltDeclarer;
-import backtype.storm.topology.TopologyBuilder;
-
-import com.alibaba.jstorm.batch.BatchTopologyBuilder;
-import com.alibaba.jstorm.batch.IBatchSpout;
-import com.alibaba.jstorm.batch.meta.MetaSpoutConfig;
-import com.alibaba.jstorm.cluster.StormConfig;
-import com.alibaba.jstorm.local.LocalCluster;
-import com.alibaba.jstorm.utils.JStormUtils;
-import com.alibaba.jstorm.utils.TimeFormat;
-
-public class BatchMetaTopology {
- private static String topologyName;
-
- private static Map conf;
-
- private static void LoadYaml(String confPath) {
-
- Yaml yaml = new Yaml();
-
- try {
- InputStream stream = new FileInputStream(confPath);
-
- conf = (Map) yaml.load(stream);
- if (conf == null || conf.isEmpty() == true) {
- throw new RuntimeException("Failed to read config file");
- }
-
- } catch (FileNotFoundException e) {
- System.out.println("No such file " + confPath);
- throw new RuntimeException("No config file");
- } catch (Exception e1) {
- e1.printStackTrace();
- throw new RuntimeException("Failed to read config file");
- }
-
- topologyName = (String) conf.get(Config.TOPOLOGY_NAME);
- return;
- }
-
- public static MetaSpoutConfig getMetaSpoutConfig(Map conf) {
- String consumerGroup = (String) conf.get("meta.consumer.group");
- String topic = (String) conf.get("meta.topic");
- String nameServer = (String) conf.get("meta.nameserver");
- String subExpress = (String) conf.get("meta.subexpress");
-
- String startTimestampStr = (String) conf.get("meta.start.timestamp");
- Long startTimeStamp = null;
- if (startTimestampStr != null) {
- Date date = TimeFormat.getSecond(startTimestampStr);
- startTimeStamp = date.getTime();
- }
-
- int batchMessageNum = JStormUtils.parseInt(
- conf.get("meta.batch.message.num"), 1024);
- int maxFailTimes = JStormUtils.parseInt(
- conf.get("meta.max.fail.times"), 10);
-
- MetaSpoutConfig ret = new MetaSpoutConfig(consumerGroup, nameServer,
- topic, subExpress);
- ret.setStartTimeStamp(startTimeStamp);
- ret.setBatchMsgNum(batchMessageNum);
- ret.setMaxFailTimes(maxFailTimes);
-
- return ret;
- }
-
- public static TopologyBuilder SetBuilder() {
-
- BatchTopologyBuilder batchTopologyBuilder = new BatchTopologyBuilder(
- topologyName);
-
- MetaSpoutConfig metaSpoutConfig = getMetaSpoutConfig(conf);
-
- BoltDeclarer rebalanceDeclarer = batchTopologyBuilder.setBolt(
- BatchMetaRebalance.BOLT_NAME, new BatchMetaRebalance(), 1);
-
- IBatchSpout batchSpout = new BatchMetaSpout(metaSpoutConfig);
- int spoutParal = JStormUtils.parseInt(
- conf.get("topology.spout.parallel"), 1);
- BoltDeclarer spoutDeclarer = batchTopologyBuilder.setSpout(
- BatchMetaSpout.SPOUT_NAME, batchSpout, spoutParal);
- spoutDeclarer.allGrouping(BatchMetaRebalance.BOLT_NAME,
- BatchMetaRebalance.REBALANCE_STREAM_ID);
-
- int boltParallel = JStormUtils.parseInt(
- conf.get("topology.bolt.parallel"), 1);
- BoltDeclarer transformDeclarer = batchTopologyBuilder.setBolt(
- TransformBolt.BOLT_NAME, new TransformBolt(), boltParallel);
- transformDeclarer.shuffleGrouping(BatchMetaSpout.SPOUT_NAME);
-
- BoltDeclarer countDeclarer = batchTopologyBuilder.setBolt(
- CountBolt.COUNT_BOLT_NAME, new CountBolt(), boltParallel);
- countDeclarer.shuffleGrouping(TransformBolt.BOLT_NAME);
-
- BoltDeclarer sumDeclarer = batchTopologyBuilder.setBolt(
- CountBolt.SUM_BOLT_NAME, new CountBolt(), boltParallel);
- sumDeclarer.shuffleGrouping(TransformBolt.BOLT_NAME);
-
- BoltDeclarer dbDeclarer = batchTopologyBuilder.setBolt(
- DBBolt.BOLT_NAME, new DBBolt(), 1);
- dbDeclarer.shuffleGrouping(CountBolt.COUNT_BOLT_NAME).shuffleGrouping(
- CountBolt.SUM_BOLT_NAME);
-
- return batchTopologyBuilder.getTopologyBuilder();
- }
-
- public static void SetLocalTopology() throws Exception {
- TopologyBuilder builder = SetBuilder();
-
- LocalCluster cluster = new LocalCluster();
- cluster.submitTopology(topologyName, conf, builder.createTopology());
-
- Thread.sleep(600000);
-
- cluster.shutdown();
- }
-
- public static void SetRemoteTopology() throws AlreadyAliveException,
- InvalidTopologyException, TopologyAssignException {
-
- TopologyBuilder builder = SetBuilder();
-
- StormSubmitter.submitTopology(topologyName, conf,
- builder.createTopology());
-
- }
-
- public static void main(String[] args) throws Exception {
-
- if (args.length < 1) {
- System.err.println("Please input parameters topology.yaml");
- System.exit(-1);
- }
-
- LoadYaml(args[0]);
-
- boolean isLocal = StormConfig.local_mode(conf);
-
- if (isLocal) {
- SetLocalTopology();
- return;
- } else {
- SetRemoteTopology();
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/e8f64d5e/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/CountBolt.java
----------------------------------------------------------------------
diff --git a/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/CountBolt.java b/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/CountBolt.java
deleted file mode 100644
index eceb4b9..0000000
--- a/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/CountBolt.java
+++ /dev/null
@@ -1,84 +0,0 @@
-package com.alibaba.jstorm.batch.example;
-
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.log4j.Logger;
-
-import backtype.storm.Config;
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.BasicOutputCollector;
-import backtype.storm.topology.IBasicBolt;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.tuple.Fields;
-import backtype.storm.tuple.Tuple;
-import backtype.storm.tuple.Values;
-import backtype.storm.utils.TimeCacheMap;
-
-import com.alibaba.jstorm.batch.BatchId;
-import com.alibaba.jstorm.batch.IPrepareCommit;
-import com.alibaba.jstorm.utils.JStormUtils;
-
-public class CountBolt implements IBasicBolt, IPrepareCommit {
- private static final long serialVersionUID = 5720810158625748042L;
-
- private static final Logger LOG = Logger.getLogger(CountBolt.class);
-
- public static final String COUNT_BOLT_NAME = "Count";
- public static final String SUM_BOLT_NAME = "Sum";
-
- private Map conf;
-
- private TimeCacheMap<BatchId, AtomicLong> counters;
-
- @Override
- public void prepare(Map stormConf, TopologyContext context) {
- this.conf = stormConf;
-
- int timeoutSeconds = JStormUtils.parseInt(
- conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS), 30);
- counters = new TimeCacheMap<BatchId, AtomicLong>(timeoutSeconds);
-
- LOG.info("Successfully do prepare");
-
- }
-
- @Override
- public void execute(Tuple input, BasicOutputCollector collector) {
- BatchId id = (BatchId) input.getValue(0);
-
- AtomicLong counter = counters.get(id);
- if (counter == null) {
- counter = new AtomicLong(0);
- counters.put(id, counter);
- }
-
- counter.incrementAndGet();
-
- }
-
- @Override
- public void cleanup() {
- LOG.info("Successfully do cleanup");
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("BatchId", "counters"));
- }
-
- @Override
- public Map<String, Object> getComponentConfiguration() {
- return null;
- }
-
- @Override
- public void prepareCommit(BatchId id, BasicOutputCollector collector) {
- AtomicLong counter = (AtomicLong) counters.remove(id);
- if (counter == null) {
- counter = new AtomicLong(0);
- }
-
- collector.emit(new Values(id, counter.get()));
- }
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/e8f64d5e/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/DBBolt.java
----------------------------------------------------------------------
diff --git a/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/DBBolt.java b/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/DBBolt.java
deleted file mode 100644
index bf9aac5..0000000
--- a/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/DBBolt.java
+++ /dev/null
@@ -1,261 +0,0 @@
-package com.alibaba.jstorm.batch.example;
-
-import java.io.Serializable;
-import java.util.Map;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.commons.lang.builder.ToStringBuilder;
-import org.apache.commons.lang.builder.ToStringStyle;
-import org.apache.log4j.Logger;
-
-import backtype.storm.Config;
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.BasicOutputCollector;
-import backtype.storm.topology.FailedException;
-import backtype.storm.topology.IBasicBolt;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.tuple.Tuple;
-import backtype.storm.utils.TimeCacheMap;
-import backtype.storm.utils.Utils;
-
-import com.alibaba.jstorm.batch.BatchId;
-import com.alibaba.jstorm.batch.ICommitter;
-import com.alibaba.jstorm.utils.JStormUtils;
-
-public class DBBolt implements IBasicBolt, ICommitter, Runnable {
- private static final long serialVersionUID = 5720810158625748043L;
-
- private static final Logger LOG = Logger.getLogger(DBBolt.class);
-
- public static final String BOLT_NAME = DBBolt.class.getSimpleName();
-
- private TimeCacheMap<BatchId, AtomicLong> counters;
- private TimeCacheMap<BatchId, AtomicLong> sums;
-
- private Map conf;
-
- private LinkedBlockingQueue<CommitedValue> revertQueue;
- private Thread revertThread;
- private AtomicBoolean isRunRevert;
-
- public void initRevert() {
- revertQueue = new LinkedBlockingQueue<DBBolt.CommitedValue>();
- isRunRevert = new AtomicBoolean(false);
- revertThread = new Thread(this);
- revertThread.start();
-
- LOG.info("Successfully init revert thread");
-
- }
-
- @Override
- public void prepare(Map stormConf, TopologyContext context) {
- this.conf = stormConf;
-
- int timeoutSeconds = JStormUtils.parseInt(
- conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS), 30);
-
- counters = new TimeCacheMap<BatchId, AtomicLong>(timeoutSeconds);
- sums = new TimeCacheMap<BatchId, AtomicLong>(timeoutSeconds);
-
- initRevert();
-
- LOG.info("Successfully do prepare");
-
- }
-
- @Override
- public void execute(Tuple input, BasicOutputCollector collector) {
- BatchId batchId = (BatchId) input.getValue(0);
-
- if (input.getSourceComponent().equals(CountBolt.COUNT_BOLT_NAME)) {
- AtomicLong counter = counters.get(batchId);
- if (counter == null) {
- counter = new AtomicLong(0);
- counters.put(batchId, counter);
- }
- long value = input.getLong(1);
-
- counter.addAndGet(value);
-
- } else if (input.getSourceComponent().equals(CountBolt.SUM_BOLT_NAME)) {
- AtomicLong sum = sums.get(batchId);
- if (sum == null) {
- sum = new AtomicLong(0);
- sums.put(batchId, sum);
- }
- long value = input.getLong(1);
-
- sum.addAndGet(value);
- } else {
- LOG.warn("Unknow source type");
- }
- }
-
- @Override
- public void cleanup() {
- LOG.info("Begin to exit ");
- isRunRevert.set(false);
- try {
- revertThread.join();
- } catch (InterruptedException e) {
-
- }
-
- LOG.info("Successfully exit ");
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- }
-
- @Override
- public Map<String, Object> getComponentConfiguration() {
- return null;
- }
-
- @Override
- public byte[] commit(BatchId id) throws FailedException {
- AtomicLong count = (AtomicLong) counters.remove(id);
- if (count == null) {
- count = new AtomicLong(0);
- }
-
- AtomicLong sum = (AtomicLong) sums.remove(id);
- if (sum == null) {
- sum = new AtomicLong(0);
- }
-
- CommitedValue commitedValue = new CommitedValue(count, sum);
-
- try {
-
- commitedValue.commit();
-
- return Utils.serialize(commitedValue);
- } catch (Exception e) {
- LOG.error("Failed to commit " + commitedValue, e);
- throw new FailedException(e);
- }
-
- }
-
- @Override
- public void revert(BatchId id, byte[] commitResult) {
- CommitedValue commitedValue = (CommitedValue) Utils
- .deserialize(commitResult);
- if (commitedValue == null) {
- LOG.warn("Failed to deserialize commited value");
- return;
- }
-
- revertQueue.offer(commitedValue);
- }
-
- @Override
- public void run() {
- LOG.info("Begin to run revert thread");
- isRunRevert.set(true);
-
- while (isRunRevert.get() == true) {
- CommitedValue commitedValue = null;
- try {
- commitedValue = revertQueue.take();
- } catch (InterruptedException e1) {
- }
- if (commitedValue == null) {
- continue;
- }
-
- try {
- commitedValue.revert();
- LOG.info("Successfully revert " + commitedValue);
- } catch (Exception e) {
- LOG.info("Failed to do revert " + commitedValue);
- JStormUtils.sleepMs(100);
- revertQueue.offer(commitedValue);
- }
- }
-
- LOG.info("Successfully quit revert thread");
- }
-
- public static class CommitedValue implements Serializable {
- private static final long serialVersionUID = 5720810158625748047L;
-
- private final AtomicLong counter;
- private final AtomicLong sum;
-
- private static AtomicLong dbCounter = new AtomicLong(0);
- private static AtomicLong dbSum = new AtomicLong(0);
-
- public CommitedValue(AtomicLong counter, AtomicLong sum) {
- this.counter = counter;
- this.sum = sum;
- }
-
- public void commit() throws Exception {
- boolean doCount = false;
- boolean doSum = false;
- try {
- long countValue = dbCounter.addAndGet(counter.get());
- doCount = true;
- long sumValue = dbSum.addAndGet(sum.get());
- doSum = true;
-
- LOG.info("Successfully commit " + this);
- LOG.info("DB counter:" + countValue + ", sum:" + sumValue);
- } catch (Exception e) {
- if (doSum) {
- dbSum.addAndGet(-sum.get());
- LOG.info("Revert sum, " + sum.get());
- }
-
- if (doCount) {
- dbCounter.addAndGet(-counter.get());
- LOG.info("Revert count," + counter.get());
- }
-
- LOG.warn(e.getCause(), e);
- throw e;
- }
-
- }
-
- public void revert() throws Exception {
- boolean doCount = false;
- boolean doSum = false;
- try {
- long countValue = dbCounter.addAndGet(-counter.get());
- doCount = true;
- long sumValue = dbSum.addAndGet(-sum.get());
- doSum = true;
-
- LOG.info("Successfully commit " + this);
- LOG.info("DB counter:" + countValue + ", sum:" + sumValue);
- } catch (Exception e) {
- if (doSum) {
- dbSum.addAndGet(sum.get());
- LOG.info("Revert sum, " + sum.get());
- }
-
- if (doCount) {
- dbCounter.addAndGet(counter.get());
- LOG.info("Revert count," + counter.get());
- }
-
- LOG.warn(e.getCause(), e);
- throw e;
- }
- }
-
- @Override
- public String toString() {
- return ToStringBuilder.reflectionToString(this,
- ToStringStyle.SHORT_PREFIX_STYLE);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/e8f64d5e/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/TransformBolt.java
----------------------------------------------------------------------
diff --git a/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/TransformBolt.java b/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/TransformBolt.java
deleted file mode 100644
index fe7c690..0000000
--- a/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/example/TransformBolt.java
+++ /dev/null
@@ -1,63 +0,0 @@
-package com.alibaba.jstorm.batch.example;
-
-import java.util.Map;
-import java.util.Random;
-
-import org.apache.log4j.Logger;
-
-import backtype.storm.task.TopologyContext;
-import backtype.storm.topology.BasicOutputCollector;
-import backtype.storm.topology.IBasicBolt;
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.tuple.Fields;
-import backtype.storm.tuple.Tuple;
-import backtype.storm.tuple.Values;
-
-import com.alibaba.jstorm.batch.BatchId;
-
-public class TransformBolt implements IBasicBolt {
- private static final long serialVersionUID = 5720810158625748044L;
-
- private static final Logger LOG = Logger.getLogger(TransformBolt.class);
-
- public static final String BOLT_NAME = TransformBolt.class.getSimpleName();
-
- private Map conf;
-
- private Random rand;
-
- @Override
- public void prepare(Map stormConf, TopologyContext context) {
- this.conf = stormConf;
- this.rand = new Random();
- rand.setSeed(System.currentTimeMillis());
-
- LOG.info("Successfully do prepare");
- }
-
- @Override
- public void execute(Tuple input, BasicOutputCollector collector) {
- BatchId id = (BatchId) input.getValue(0);
-
- long value = rand.nextInt(100);
-
- collector.emit(new Values(id, value));
-
- }
-
- @Override
- public void cleanup() {
- LOG.info("Successfully do cleanup");
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("BatchId", "Value"));
- }
-
- @Override
- public Map<String, Object> getComponentConfiguration() {
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/e8f64d5e/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/meta/MetaSimpleClient.java
----------------------------------------------------------------------
diff --git a/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/meta/MetaSimpleClient.java b/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/meta/MetaSimpleClient.java
deleted file mode 100644
index ee908d1..0000000
--- a/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/meta/MetaSimpleClient.java
+++ /dev/null
@@ -1,340 +0,0 @@
-package com.alibaba.jstorm.batch.meta;
-
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.log4j.Logger;
-
-import backtype.storm.topology.FailedException;
-
-import com.alibaba.jstorm.batch.BatchId;
-import com.alibaba.jstorm.batch.ICommitter;
-import com.alibaba.jstorm.batch.util.BatchDef;
-import com.alibaba.jstorm.cluster.ClusterState;
-import com.alibaba.jstorm.utils.JStormUtils;
-import com.alibaba.rocketmq.client.consumer.PullResult;
-import com.alibaba.rocketmq.client.consumer.PullStatus;
-import com.alibaba.rocketmq.client.exception.MQClientException;
-import com.alibaba.rocketmq.common.message.MessageExt;
-import com.alibaba.rocketmq.common.message.MessageQueue;
-import com.taobao.metaq.client.MetaPullConsumer;
-
-public class MetaSimpleClient implements ICommitter{
- private static final Logger LOG = Logger.getLogger(MetaSimpleClient.class);
-
- private final MetaSpoutConfig metaSpoutConfig;
-
- private final int taskIndex;
- private final int taskParallel;
- private int oneQueueFetchSize;
-
- private Map<MessageQueue, Long> currentOffsets;
- private Map<MessageQueue, Long> frontOffsets;
- private Map<MessageQueue, Long> backendOffset;
-
- private MetaPullConsumer consumer;
-
- private static String nameServer;
-
- private final ClusterState zkClient;
-
- public MetaSimpleClient(MetaSpoutConfig config,
- ClusterState zkClient,
- int taskIndex,
- int taskParall) {
- this.metaSpoutConfig = config;
- this.zkClient = zkClient;
- this.taskIndex = taskIndex;
- this.taskParallel = taskParall;
- }
-
- public void initMetaConsumer() throws MQClientException {
- LOG.info("MetaSpoutConfig:" + metaSpoutConfig);
-
- consumer = new MetaPullConsumer(metaSpoutConfig.getConsumerGroup());
- consumer.setInstanceName(taskIndex + "." + JStormUtils.process_pid());
-
- if (metaSpoutConfig.getNameServer() != null) {
- // this is for alipay
- if (nameServer == null) {
- nameServer = metaSpoutConfig.getNameServer();
-
- System.setProperty("rocketmq.namesrv.domain",
- metaSpoutConfig.getNameServer());
- }else if (metaSpoutConfig.getNameServer().equals(nameServer) == false) {
- throw new RuntimeException("Different nameserver address in the same worker "
- + nameServer + ":" + metaSpoutConfig.getNameServer());
-
- }
- }
-
- consumer.start();
-
- LOG.info("Successfully start meta consumer");
- }
-
- public int getOneQueueFetchSize() {
- int batchSize = metaSpoutConfig.getBatchMsgNum();
-
- int oneFetchSize = batchSize/(taskParallel * currentOffsets.size());
- if (oneFetchSize <= 0) {
- oneFetchSize = 1;
- }
-
- LOG.info("One queue fetch size:" + oneFetchSize);
- return oneFetchSize;
- }
-
- public void init() throws Exception {
- initMetaConsumer();
-
- frontOffsets = initOffset();
- currentOffsets = frontOffsets;
- backendOffset = new HashMap<MessageQueue, Long>();
- backendOffset.putAll(frontOffsets);
-
- oneQueueFetchSize = getOneQueueFetchSize();
- }
-
-
- protected Set<MessageQueue> getMQ() throws MQClientException {
- Set<MessageQueue> mqs = consumer.
- fetchSubscribeMessageQueues(metaSpoutConfig.getTopic());
- if (taskParallel > mqs.size()) {
- throw new RuntimeException("Two much task to fetch " + metaSpoutConfig);
- }
-
- List<MessageQueue> mqList = JStormUtils.mk_list(mqs);
- Set<MessageQueue> ret = new HashSet<MessageQueue>();
- for (int i = taskIndex; i < mqList.size(); i += taskParallel) {
- ret.add(mqList.get(i));
- }
-
- if (ret.size() == 0) {
- throw new RuntimeException("No meta queue need to be consume");
- }
- return ret;
- }
-
- protected Map<MessageQueue, Long> initOffset() throws MQClientException {
- Set<MessageQueue> queues = getMQ();
-
- Map<MessageQueue, Long> ret = new HashMap<MessageQueue, Long>();
- Set<MessageQueue> noOffsetQueues = new HashSet<MessageQueue>();
-
-
- if (metaSpoutConfig.getStartTimeStamp() != null) {
- Long timeStamp = metaSpoutConfig.getStartTimeStamp();
-
-
- for (MessageQueue mq : queues) {
- long offset = consumer.searchOffset(mq, timeStamp);
- if (offset >= 0) {
- LOG.info("Successfully get " + mq + " offset of timestamp " + new Date(timeStamp));
- ret.put(mq, offset);
- }else {
- LOG.info("Failed to get " + mq + " offset of timestamp " + new Date(timeStamp));
- noOffsetQueues.add(mq);
- }
- }
- }else {
- noOffsetQueues.addAll(queues);
- }
-
- if (noOffsetQueues.size() == 0) {
- return ret;
- }
-
- for (MessageQueue mq : noOffsetQueues) {
- long offset = getOffsetFromZk(mq);
-
- ret.put(mq, offset);
- }
-
- return ret;
- }
-
-
- protected String getZkPath(MessageQueue mq) {
- StringBuffer sb = new StringBuffer();
- sb.append(BatchDef.ZK_SEPERATOR);
- sb.append(metaSpoutConfig.getConsumerGroup());
- sb.append(BatchDef.ZK_SEPERATOR);
- sb.append(mq.getBrokerName());
- sb.append(BatchDef.ZK_SEPERATOR);
- sb.append(mq.getQueueId());
-
- return sb.toString();
- }
-
- protected long getOffsetFromZk(MessageQueue mq) {
- String path = getZkPath(mq);
-
- try {
- if (zkClient.node_existed(path, false) == false) {
- LOG.info("No zk node of " + path);
- return 0;
- }
-
- byte[] data = zkClient.get_data(path, false);
- String value = new String(data);
-
- long ret = Long.valueOf(value);
- return ret;
- }catch (Exception e) {
- LOG.warn("Failed to get offset,", e);
- return 0;
- }
- }
-
- protected void updateOffsetToZk(MessageQueue mq, Long offset) throws Exception {
- String path = getZkPath(mq);
- byte[] data = String.valueOf(offset).getBytes();
- zkClient.set_data(path, data);
-
- }
-
- protected void updateOffsetToZk(Map<MessageQueue, Long> mqs) throws Exception{
- for (Entry<MessageQueue, Long> entry : mqs.entrySet()) {
- MessageQueue mq = entry.getKey();
- Long offset = entry.getValue();
-
- updateOffsetToZk(mq, offset);
- }
-
- LOG.info("Update zk offset," + mqs);
- }
-
- protected void switchOffsetMap() {
- Map<MessageQueue, Long> tmp = frontOffsets;
-
- frontOffsets = backendOffset;
- backendOffset = tmp;
-
- currentOffsets = frontOffsets;
- }
-
- @Override
- public byte[] commit(BatchId id) throws FailedException {
- try {
- updateOffsetToZk(currentOffsets);
- switchOffsetMap();
- }catch(Exception e) {
- LOG.warn("Failed to update offset to ZK", e);
- throw new FailedException(e);
- }
-
- return null;
- }
-
- @Override
- public void revert(BatchId id, byte[] commitResult) {
- try {
- switchOffsetMap();
- updateOffsetToZk(currentOffsets);
-
- }catch(Exception e) {
- LOG.warn("Failed to update offset to ZK", e);
- throw new FailedException(e);
- }
- }
-
- /**
- * rebalanceMqList must run after commit
- *
- * @throws MQClientException
- */
- public void rebalanceMqList() throws Exception {
- LOG.info("Begin to do rebalance operation");
- Set<MessageQueue> newMqs = getMQ();
-
- Set<MessageQueue> oldMqs = currentOffsets.keySet();
-
- if (oldMqs.equals(newMqs) == true) {
- LOG.info("No change of meta queues " + newMqs);
- return ;
- }
-
- Set<MessageQueue> removeMqs = new HashSet<MessageQueue>();
- removeMqs.addAll(oldMqs);
- removeMqs.removeAll(newMqs);
-
- Set<MessageQueue> addMqs = new HashSet<MessageQueue>();
- addMqs.addAll(newMqs);
- addMqs.removeAll(oldMqs);
-
- LOG.info("Remove " + removeMqs);
- for (MessageQueue mq : removeMqs) {
- Long offset = frontOffsets.remove(mq);
- updateOffsetToZk(mq, offset);
-
- backendOffset.remove(mq);
-
- }
-
- LOG.info("Add " + addMqs);
- for (MessageQueue mq : addMqs) {
- long offset = getOffsetFromZk(mq);
- frontOffsets.put(mq, offset);
- backendOffset.put(mq, offset);
- }
- }
-
- public List<MessageExt> fetchOneBatch() {
- List<MessageExt> ret = new ArrayList<MessageExt>();
-
-
- String subexpress = metaSpoutConfig.getSubExpress();
- for(Entry<MessageQueue, Long>entry : currentOffsets.entrySet()) {
- MessageQueue mq = entry.getKey();
- Long offset = entry.getValue();
-
-
- int fetchSize = 0;
- int oneFetchSize = Math.min(oneQueueFetchSize, 32);
-
- while(fetchSize < oneQueueFetchSize) {
-
- PullResult pullResult = null;
- try {
- pullResult = consumer.pullBlockIfNotFound(mq, subexpress, offset, oneFetchSize);
- offset = pullResult.getNextBeginOffset();
- PullStatus status = pullResult.getPullStatus();
- if (status == PullStatus.FOUND) {
- List<MessageExt> msgList = pullResult.getMsgFoundList();
- ret.addAll(msgList);
- fetchSize += msgList.size();
- continue;
- }else if (status == PullStatus.NO_MATCHED_MSG) {
- continue;
- }else if (status == PullStatus.NO_NEW_MSG ) {
- break;
- }else if (status == PullStatus.OFFSET_ILLEGAL) {
- break;
- }else {
-
- break;
- }
- }
- catch (Exception e) {
- LOG.warn("Failed to fetch messages of " + mq + ":" + offset, e);
- break;
- }
- }
-
- backendOffset.put(mq, offset);
- }
-
- return ret;
- }
-
- public void cleanup() {
- consumer.shutdown();
- }
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/e8f64d5e/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/meta/MetaSpoutConfig.java
----------------------------------------------------------------------
diff --git a/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/meta/MetaSpoutConfig.java b/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/meta/MetaSpoutConfig.java
deleted file mode 100644
index ac25825..0000000
--- a/jstorm-utility/transaction_meta_spout/src/main/java/com/alibaba/jstorm/batch/meta/MetaSpoutConfig.java
+++ /dev/null
@@ -1,119 +0,0 @@
-package com.alibaba.jstorm.batch.meta;
-
-import java.io.Serializable;
-import java.util.Properties;
-
-import org.apache.commons.lang.builder.ToStringBuilder;
-import org.apache.commons.lang.builder.ToStringStyle;
-
-/**
- * Meta Spout Setting
- *
- * All needed configs must prepare before submit topopoly
- *
- * @author longda
- */
-public class MetaSpoutConfig implements Serializable {
-
- private static final long serialVersionUID = 4157424979688593280L;
-
- private final String consumerGroup;
-
- /**
- * Alipay need set nameServer, taobao don't need set this field
- */
- private final String nameServer;
-
- private final String topic;
-
- private final String subExpress;
-
- /**
- * The max allowed failures for one single message, skip the failure message
- * if excesses
- *
- * -1 means try again until success
- */
- private int maxFailTimes = DEFAULT_FAIL_TIME;
- public static final int DEFAULT_FAIL_TIME = 10;
-
- /**
- * One batch's message number
- *
- */
- private int batchMsgNum = DEFAULT_BATCH_MSG_NUM;
- public static final int DEFAULT_BATCH_MSG_NUM = 256;
-
- /**
- * Consumer start time Null means start from the last consumption
- * time(CONSUME_FROM_LAST_OFFSET)
- *
- */
- private Long startTimeStamp;
-
- private Properties peroperties;
-
- public MetaSpoutConfig(String consumerGroup, String nameServer,
- String topic, String subExpress) {
- super();
- this.consumerGroup = consumerGroup;
- this.nameServer = nameServer;
- this.topic = topic;
- this.subExpress = subExpress;
- }
-
- public int getMaxFailTimes() {
- return maxFailTimes;
- }
-
- public void setMaxFailTimes(int maxFailTimes) {
- this.maxFailTimes = maxFailTimes;
- }
-
- public Long getStartTimeStamp() {
- return startTimeStamp;
- }
-
- public void setStartTimeStamp(Long startTimeStamp) {
- this.startTimeStamp = startTimeStamp;
- }
-
- public Properties getPeroperties() {
- return peroperties;
- }
-
- public void setPeroperties(Properties peroperties) {
- this.peroperties = peroperties;
- }
-
- public String getConsumerGroup() {
- return consumerGroup;
- }
-
- public String getNameServer() {
- return nameServer;
- }
-
- public String getTopic() {
- return topic;
- }
-
- public String getSubExpress() {
- return subExpress;
- }
-
- public int getBatchMsgNum() {
- return batchMsgNum;
- }
-
- public void setBatchMsgNum(int batchMsgNum) {
- this.batchMsgNum = batchMsgNum;
- }
-
- @Override
- public String toString() {
- return ToStringBuilder.reflectionToString(this,
- ToStringStyle.SHORT_PREFIX_STYLE);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/storm/blob/e8f64d5e/jstorm-utility/transaction_meta_spout/src/main/resources/metaspout.default.prop
----------------------------------------------------------------------
diff --git a/jstorm-utility/transaction_meta_spout/src/main/resources/metaspout.default.prop b/jstorm-utility/transaction_meta_spout/src/main/resources/metaspout.default.prop
deleted file mode 100644
index 8b6e45d..0000000
--- a/jstorm-utility/transaction_meta_spout/src/main/resources/metaspout.default.prop
+++ /dev/null
@@ -1,15 +0,0 @@
-meta.consumer.group=test1Groupaa
-meta.topic=tlog-test
-meta.subexpress=*
-meta.fetch.max.msg=128
-
-topology.name=test_meta_spout
-topology.version=1.0.0
-topology.workers=5
-
-topology.spout.pending=100
-topology.spout.parallel=1
-topology.writer.parallel=1
-
-topology.acker.executors=1
-topology.use.transaction=false