You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by pt...@apache.org on 2014/08/08 21:48:32 UTC
[21/50] [abbrv] git commit: Implement TimedRotationPolicy
Implement TimedRotationPolicy
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/be5eae3b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/be5eae3b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/be5eae3b
Branch: refs/heads/master
Commit: be5eae3bcc07cfbaf49c19c57da42dac1351a3e3
Parents: f227830
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Jun 26 17:04:02 2014 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jun 26 17:04:02 2014 -0400
----------------------------------------------------------------------
.../storm/hdfs/bolt/AbstractHdfsBolt.java | 22 +++++++-
.../hdfs/bolt/rotation/TimedRotationPolicy.java | 55 ++++++++++++++++++++
.../apache/storm/hdfs/trident/HdfsState.java | 25 +++++++--
.../trident/rotation/TimedRotationPolicy.java | 54 +++++++++++++++++++
.../storm/hdfs/bolt/HdfsFileTopology.java | 3 +-
5 files changed, 152 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/be5eae3b/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java b/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
index 837b965..f260598 100644
--- a/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
+++ b/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.storm.hdfs.bolt.format.FileNameFormat;
import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
+import org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy;
import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
import org.apache.storm.hdfs.common.rotation.RotationAction;
import org.apache.storm.hdfs.common.security.HdfsSecurityUtil;
@@ -35,6 +36,8 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
public abstract class AbstractHdfsBolt extends BaseRichBolt {
private static final Logger LOG = LoggerFactory.getLogger(AbstractHdfsBolt.class);
@@ -49,7 +52,8 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
protected int rotation = 0;
protected String fsUrl;
protected String configKey;
- protected Object writeLock;
+ protected transient Object writeLock;
+ protected transient Timer rotationTimer; // only used for TimedRotationPolicy
protected transient Configuration hdfsConfig;
@@ -104,6 +108,22 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
} catch (Exception e){
throw new RuntimeException("Error preparing HdfsBolt: " + e.getMessage(), e);
}
+
+ if(this.rotationPolicy instanceof TimedRotationPolicy){
+ long interval = ((TimedRotationPolicy)this.rotationPolicy).getInterval();
+ this.rotationTimer = new Timer(true);
+ TimerTask task = new TimerTask() {
+ @Override
+ public void run() {
+ try {
+ rotateOutputFile();
+ } catch(IOException e){
+ LOG.warn("IOException during scheduled file rotation.", e);
+ }
+ }
+ };
+ this.rotationTimer.scheduleAtFixedRate(task, interval, interval);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/be5eae3b/src/main/java/org/apache/storm/hdfs/bolt/rotation/TimedRotationPolicy.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/storm/hdfs/bolt/rotation/TimedRotationPolicy.java b/src/main/java/org/apache/storm/hdfs/bolt/rotation/TimedRotationPolicy.java
new file mode 100644
index 0000000..6cf8f3d
--- /dev/null
+++ b/src/main/java/org/apache/storm/hdfs/bolt/rotation/TimedRotationPolicy.java
@@ -0,0 +1,55 @@
+package org.apache.storm.hdfs.bolt.rotation;
+
+import backtype.storm.tuple.Tuple;
+
+public class TimedRotationPolicy implements FileRotationPolicy {
+
+ public static enum TimeUnit {
+
+ SECONDS((long)1000),
+ MINUTES((long)1000*60),
+ HOURS((long)1000*60*60),
+ DAYS((long)1000*60*60*24);
+
+ private long milliSeconds;
+
+ private TimeUnit(long milliSeconds){
+ this.milliSeconds = milliSeconds;
+ }
+
+ public long getMilliSeconds(){
+ return milliSeconds;
+ }
+ }
+
+ private long interval;
+
+ public TimedRotationPolicy(float count, TimeUnit units){
+ this.interval = (long)(count * units.getMilliSeconds());
+ }
+
+
+ /**
+ * Called for every tuple the HdfsBolt executes.
+ *
+ * @param tuple The tuple executed.
+ * @param offset current offset of file being written
+ * @return true if a file rotation should be performed
+ */
+ @Override
+ public boolean mark(Tuple tuple, long offset) {
+ return false;
+ }
+
+ /**
+ * Called after the HdfsBolt rotates a file.
+ */
+ @Override
+ public void reset() {
+
+ }
+
+ public long getInterval(){
+ return this.interval;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/be5eae3b/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java b/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
index 1c0617a..490e488 100644
--- a/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
+++ b/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
@@ -16,6 +16,7 @@ import org.apache.storm.hdfs.trident.format.RecordFormat;
import org.apache.storm.hdfs.trident.format.SequenceFormat;
import org.apache.storm.hdfs.trident.rotation.FileRotationPolicy;
+import org.apache.storm.hdfs.trident.rotation.TimedRotationPolicy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import storm.trident.operation.TridentCollector;
@@ -25,10 +26,7 @@ import storm.trident.tuple.TridentTuple;
import java.io.IOException;
import java.io.Serializable;
import java.net.URI;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
public class HdfsState implements State {
@@ -43,7 +41,8 @@ public class HdfsState implements State {
protected int rotation = 0;
protected transient Configuration hdfsConfig;
protected ArrayList<RotationAction> rotationActions = new ArrayList<RotationAction>();
- protected Object writeLock;
+ protected transient Object writeLock;
+ protected transient Timer rotationTimer;
abstract void closeOutputFile() throws IOException;
@@ -95,6 +94,22 @@ public class HdfsState implements State {
} catch (Exception e){
throw new RuntimeException("Error preparing HdfsState: " + e.getMessage(), e);
}
+
+ if(this.rotationPolicy instanceof TimedRotationPolicy){
+ long interval = ((TimedRotationPolicy)this.rotationPolicy).getInterval();
+ this.rotationTimer = new Timer(true);
+ TimerTask task = new TimerTask() {
+ @Override
+ public void run() {
+ try {
+ rotateOutputFile();
+ } catch(IOException e){
+ LOG.warn("IOException during scheduled file rotation.", e);
+ }
+ }
+ };
+ this.rotationTimer.scheduleAtFixedRate(task, interval, interval);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/be5eae3b/src/main/java/org/apache/storm/hdfs/trident/rotation/TimedRotationPolicy.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/storm/hdfs/trident/rotation/TimedRotationPolicy.java b/src/main/java/org/apache/storm/hdfs/trident/rotation/TimedRotationPolicy.java
new file mode 100644
index 0000000..b2f3e3a
--- /dev/null
+++ b/src/main/java/org/apache/storm/hdfs/trident/rotation/TimedRotationPolicy.java
@@ -0,0 +1,54 @@
+package org.apache.storm.hdfs.trident.rotation;
+
+import storm.trident.tuple.TridentTuple;
+
+
+public class TimedRotationPolicy implements FileRotationPolicy {
+
+ public static enum TimeUnit {
+
+ SECONDS((long)1000),
+ MINUTES((long)1000*60),
+ HOURS((long)1000*60*60),
+ DAYS((long)1000*60*60*24);
+
+ private long milliSeconds;
+
+ private TimeUnit(long milliSeconds){
+ this.milliSeconds = milliSeconds;
+ }
+
+ public long getMilliSeconds(){
+ return milliSeconds;
+ }
+ }
+
+ private long interval;
+
+ public TimedRotationPolicy(float count, TimeUnit units){
+ this.interval = (long)(count * units.getMilliSeconds());
+ }
+ /**
+ * Called for every tuple the HdfsBolt executes.
+ *
+ * @param tuple The tuple executed.
+ * @param offset current offset of file being written
+ * @return true if a file rotation should be performed
+ */
+ @Override
+ public boolean mark(TridentTuple tuple, long offset) {
+ return false;
+ }
+
+ /**
+ * Called after the HdfsBolt rotates a file.
+ */
+ @Override
+ public void reset() {
+
+ }
+
+ public long getInterval(){
+ return this.interval;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/be5eae3b/src/test/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java b/src/test/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
index 48c629e..4fc6fb3 100644
--- a/src/test/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
+++ b/src/test/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
@@ -37,6 +37,7 @@ import org.apache.storm.hdfs.bolt.format.RecordFormat;
import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy;
import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy.Units;
+import org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy;
import org.apache.storm.hdfs.bolt.sync.CountSyncPolicy;
import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
import org.apache.storm.hdfs.common.rotation.MoveFileAction;
@@ -64,7 +65,7 @@ public class HdfsFileTopology {
SyncPolicy syncPolicy = new CountSyncPolicy(1000);
// rotate files when they reach 5MB
- FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, Units.MB);
+ FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, TimedRotationPolicy.TimeUnit.MINUTES);
FileNameFormat fileNameFormat = new DefaultFileNameFormat()
.withPath("/foo/")