You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by pt...@apache.org on 2014/08/08 21:48:32 UTC

[21/50] [abbrv] git commit: Implement TimedRotationPolicy

Implement TimedRotationPolicy


Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/be5eae3b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/be5eae3b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/be5eae3b

Branch: refs/heads/master
Commit: be5eae3bcc07cfbaf49c19c57da42dac1351a3e3
Parents: f227830
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Jun 26 17:04:02 2014 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jun 26 17:04:02 2014 -0400

----------------------------------------------------------------------
 .../storm/hdfs/bolt/AbstractHdfsBolt.java       | 22 +++++++-
 .../hdfs/bolt/rotation/TimedRotationPolicy.java | 55 ++++++++++++++++++++
 .../apache/storm/hdfs/trident/HdfsState.java    | 25 +++++++--
 .../trident/rotation/TimedRotationPolicy.java   | 54 +++++++++++++++++++
 .../storm/hdfs/bolt/HdfsFileTopology.java       |  3 +-
 5 files changed, 152 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/be5eae3b/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java b/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
index 837b965..f260598 100644
--- a/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
+++ b/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.storm.hdfs.bolt.format.FileNameFormat;
 import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
+import org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy;
 import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
 import org.apache.storm.hdfs.common.rotation.RotationAction;
 import org.apache.storm.hdfs.common.security.HdfsSecurityUtil;
@@ -35,6 +36,8 @@ import org.slf4j.LoggerFactory;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
 
 public abstract class AbstractHdfsBolt extends BaseRichBolt {
     private static final Logger LOG = LoggerFactory.getLogger(AbstractHdfsBolt.class);
@@ -49,7 +52,8 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
     protected int rotation = 0;
     protected String fsUrl;
     protected String configKey;
-    protected Object writeLock;
+    protected transient Object writeLock;
+    protected transient Timer rotationTimer; // only used for TimedRotationPolicy
 
     protected transient Configuration hdfsConfig;
 
@@ -104,6 +108,22 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
         } catch (Exception e){
             throw new RuntimeException("Error preparing HdfsBolt: " + e.getMessage(), e);
         }
+
+        if(this.rotationPolicy instanceof TimedRotationPolicy){
+            long interval = ((TimedRotationPolicy)this.rotationPolicy).getInterval();
+            this.rotationTimer = new Timer(true);
+            TimerTask task = new TimerTask() {
+                @Override
+                public void run() {
+                    try {
+                        rotateOutputFile();
+                    } catch(IOException e){
+                        LOG.warn("IOException during scheduled file rotation.", e);
+                    }
+                }
+            };
+            this.rotationTimer.scheduleAtFixedRate(task, interval, interval);
+        }
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/be5eae3b/src/main/java/org/apache/storm/hdfs/bolt/rotation/TimedRotationPolicy.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/storm/hdfs/bolt/rotation/TimedRotationPolicy.java b/src/main/java/org/apache/storm/hdfs/bolt/rotation/TimedRotationPolicy.java
new file mode 100644
index 0000000..6cf8f3d
--- /dev/null
+++ b/src/main/java/org/apache/storm/hdfs/bolt/rotation/TimedRotationPolicy.java
@@ -0,0 +1,55 @@
+package org.apache.storm.hdfs.bolt.rotation;
+
+import backtype.storm.tuple.Tuple;
+
+public class TimedRotationPolicy implements FileRotationPolicy {
+
+    public static enum TimeUnit {
+
+        SECONDS((long)1000),
+        MINUTES((long)1000*60),
+        HOURS((long)1000*60*60),
+        DAYS((long)1000*60*60*24);
+
+        private long milliSeconds;
+
+        private TimeUnit(long milliSeconds){
+            this.milliSeconds = milliSeconds;
+        }
+
+        public long getMilliSeconds(){
+            return milliSeconds;
+        }
+    }
+
+    private long interval;
+
+    public TimedRotationPolicy(float count, TimeUnit units){
+        this.interval = (long)(count * units.getMilliSeconds());
+    }
+
+
+    /**
+     * Called for every tuple the HdfsBolt executes.
+     *
+     * @param tuple  The tuple executed.
+     * @param offset current offset of file being written
+     * @return true if a file rotation should be performed
+     */
+    @Override
+    public boolean mark(Tuple tuple, long offset) {
+        return false;
+    }
+
+    /**
+     * Called after the HdfsBolt rotates a file.
+     */
+    @Override
+    public void reset() {
+
+    }
+
+    public long getInterval(){
+        return this.interval;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/be5eae3b/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java b/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
index 1c0617a..490e488 100644
--- a/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
+++ b/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
@@ -16,6 +16,7 @@ import org.apache.storm.hdfs.trident.format.RecordFormat;
 import org.apache.storm.hdfs.trident.format.SequenceFormat;
 import org.apache.storm.hdfs.trident.rotation.FileRotationPolicy;
 
+import org.apache.storm.hdfs.trident.rotation.TimedRotationPolicy;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import storm.trident.operation.TridentCollector;
@@ -25,10 +26,7 @@ import storm.trident.tuple.TridentTuple;
 import java.io.IOException;
 import java.io.Serializable;
 import java.net.URI;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 public class HdfsState implements State {
 
@@ -43,7 +41,8 @@ public class HdfsState implements State {
         protected int rotation = 0;
         protected transient Configuration hdfsConfig;
         protected ArrayList<RotationAction> rotationActions = new ArrayList<RotationAction>();
-        protected Object writeLock;
+        protected transient Object writeLock;
+        protected transient Timer rotationTimer;
 
         abstract void closeOutputFile() throws IOException;
 
@@ -95,6 +94,22 @@ public class HdfsState implements State {
             } catch (Exception e){
                 throw new RuntimeException("Error preparing HdfsState: " + e.getMessage(), e);
             }
+
+            if(this.rotationPolicy instanceof TimedRotationPolicy){
+                long interval = ((TimedRotationPolicy)this.rotationPolicy).getInterval();
+                this.rotationTimer = new Timer(true);
+                TimerTask task = new TimerTask() {
+                    @Override
+                    public void run() {
+                        try {
+                            rotateOutputFile();
+                        } catch(IOException e){
+                            LOG.warn("IOException during scheduled file rotation.", e);
+                        }
+                    }
+                };
+                this.rotationTimer.scheduleAtFixedRate(task, interval, interval);
+            }
         }
 
     }

http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/be5eae3b/src/main/java/org/apache/storm/hdfs/trident/rotation/TimedRotationPolicy.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/storm/hdfs/trident/rotation/TimedRotationPolicy.java b/src/main/java/org/apache/storm/hdfs/trident/rotation/TimedRotationPolicy.java
new file mode 100644
index 0000000..b2f3e3a
--- /dev/null
+++ b/src/main/java/org/apache/storm/hdfs/trident/rotation/TimedRotationPolicy.java
@@ -0,0 +1,54 @@
+package org.apache.storm.hdfs.trident.rotation;
+
+import storm.trident.tuple.TridentTuple;
+
+
+public class TimedRotationPolicy implements FileRotationPolicy {
+
+    public static enum TimeUnit {
+
+        SECONDS((long)1000),
+        MINUTES((long)1000*60),
+        HOURS((long)1000*60*60),
+        DAYS((long)1000*60*60*24);
+
+        private long milliSeconds;
+
+        private TimeUnit(long milliSeconds){
+            this.milliSeconds = milliSeconds;
+        }
+
+        public long getMilliSeconds(){
+            return milliSeconds;
+        }
+    }
+
+    private long interval;
+
+    public TimedRotationPolicy(float count, TimeUnit units){
+        this.interval = (long)(count * units.getMilliSeconds());
+    }
+    /**
+     * Called for every tuple the HdfsBolt executes.
+     *
+     * @param tuple  The tuple executed.
+     * @param offset current offset of file being written
+     * @return true if a file rotation should be performed
+     */
+    @Override
+    public boolean mark(TridentTuple tuple, long offset) {
+        return false;
+    }
+
+    /**
+     * Called after the HdfsBolt rotates a file.
+     */
+    @Override
+    public void reset() {
+
+    }
+
+    public long getInterval(){
+        return this.interval;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/be5eae3b/src/test/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java b/src/test/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
index 48c629e..4fc6fb3 100644
--- a/src/test/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
+++ b/src/test/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
@@ -37,6 +37,7 @@ import org.apache.storm.hdfs.bolt.format.RecordFormat;
 import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
 import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy;
 import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy.Units;
+import org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy;
 import org.apache.storm.hdfs.bolt.sync.CountSyncPolicy;
 import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
 import org.apache.storm.hdfs.common.rotation.MoveFileAction;
@@ -64,7 +65,7 @@ public class HdfsFileTopology {
         SyncPolicy syncPolicy = new CountSyncPolicy(1000);
 
         // rotate files when they reach 5MB
-        FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, Units.MB);
+        FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, TimedRotationPolicy.TimeUnit.MINUTES);
 
         FileNameFormat fileNameFormat = new DefaultFileNameFormat()
                 .withPath("/foo/")