You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by pt...@apache.org on 2014/08/08 21:48:31 UTC

[20/50] [abbrv] git commit: implement locking for file writes/rotation

implement locking for file writes/rotation


Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/f227830f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/f227830f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/f227830f

Branch: refs/heads/master
Commit: f227830fdbe14290c6870835b3aab6f701dbdddb
Parents: 3509e24
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Jun 26 16:06:30 2014 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jun 26 16:06:50 2014 -0400

----------------------------------------------------------------------
 .../storm/hdfs/bolt/AbstractHdfsBolt.java       | 20 ++++---
 .../org/apache/storm/hdfs/bolt/HdfsBolt.java    | 22 +++----
 .../storm/hdfs/bolt/SequenceFileBolt.java       | 20 ++++---
 .../apache/storm/hdfs/trident/HdfsState.java    | 61 +++++++++++---------
 .../storm/hdfs/bolt/HdfsFileTopology.java       |  4 +-
 5 files changed, 73 insertions(+), 54 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/f227830f/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java b/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
index 8d696f4..837b965 100644
--- a/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
+++ b/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
@@ -49,21 +49,24 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
     protected int rotation = 0;
     protected String fsUrl;
     protected String configKey;
+    protected Object writeLock;
 
     protected transient Configuration hdfsConfig;
 
     protected void rotateOutputFile() throws IOException {
         LOG.info("Rotating output file...");
         long start = System.currentTimeMillis();
-        closeOutputFile();
-        this.rotation++;
-
-        Path newFile = createOutputFile();
-        LOG.info("Performing {} file rotation actions.", this.rotationActions.size());
-        for(RotationAction action : this.rotationActions){
-            action.execute(this.fs, this.currentFile);
+        synchronized (this.writeLock) {
+            closeOutputFile();
+            this.rotation++;
+
+            Path newFile = createOutputFile();
+            LOG.info("Performing {} file rotation actions.", this.rotationActions.size());
+            for (RotationAction action : this.rotationActions) {
+                action.execute(this.fs, this.currentFile);
+            }
+            this.currentFile = newFile;
         }
-        this.currentFile = newFile;
         long time = System.currentTimeMillis() - start;
         LOG.info("File rotation took {} ms.", time);
     }
@@ -75,6 +78,7 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
      * @param collector
      */
     public final void prepare(Map conf, TopologyContext topologyContext, OutputCollector collector){
+        this.writeLock = new Object();
         if (this.syncPolicy == null) throw new IllegalStateException("SyncPolicy must be specified.");
         if (this.rotationPolicy == null) throw new IllegalStateException("RotationPolicy must be specified.");
         if (this.fsUrl == null) {

http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/f227830f/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java b/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java
index 5c36c65..a416357 100644
--- a/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java
+++ b/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java
@@ -90,22 +90,24 @@ public class HdfsBolt extends AbstractHdfsBolt{
     public void execute(Tuple tuple) {
         try {
             byte[] bytes = this.format.format(tuple);
-            out.write(bytes);
-            this.offset += bytes.length;
-
-            if(this.syncPolicy.mark(tuple, this.offset)){
-                if(this.out instanceof HdfsDataOutputStream){
-                    ((HdfsDataOutputStream)this.out).hsync(EnumSet.of(SyncFlag.UPDATE_LENGTH));
-                } else {
-                    this.out.hsync();
+            synchronized (this.writeLock) {
+                out.write(bytes);
+                this.offset += bytes.length;
+
+                if (this.syncPolicy.mark(tuple, this.offset)) {
+                    if (this.out instanceof HdfsDataOutputStream) {
+                        ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(SyncFlag.UPDATE_LENGTH));
+                    } else {
+                        this.out.hsync();
+                    }
+                    this.syncPolicy.reset();
                 }
-                this.syncPolicy.reset();
             }
 
             this.collector.ack(tuple);
 
             if(this.rotationPolicy.mark(tuple, this.offset)){
-                rotateOutputFile();
+                rotateOutputFile(); // synchronized
                 this.offset = 0;
                 this.rotationPolicy.reset();
             }

http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/f227830f/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java b/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java
index 6bc5dd2..fc9bb4f 100644
--- a/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java
+++ b/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java
@@ -106,16 +106,20 @@ public class SequenceFileBolt extends AbstractHdfsBolt {
     @Override
     public void execute(Tuple tuple) {
         try {
-            this.writer.append(this.format.key(tuple), this.format.value(tuple));
-            long offset = this.writer.getLength();
-            this.collector.ack(tuple);
-
-            if (this.syncPolicy.mark(tuple, offset)) {
-                this.writer.hsync();
-                this.syncPolicy.reset();
+            long offset;
+            synchronized (this.writeLock) {
+                this.writer.append(this.format.key(tuple), this.format.value(tuple));
+                offset = this.writer.getLength();
+
+                if (this.syncPolicy.mark(tuple, offset)) {
+                    this.writer.hsync();
+                    this.syncPolicy.reset();
+                }
             }
+
+            this.collector.ack(tuple);
             if (this.rotationPolicy.mark(tuple, offset)) {
-                rotateOutputFile();
+                rotateOutputFile(); // synchronized
                 this.rotationPolicy.reset();
             }
         } catch (IOException e) {

http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/f227830f/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java b/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
index 01793b9..1c0617a 100644
--- a/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
+++ b/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
@@ -43,6 +43,7 @@ public class HdfsState implements State {
         protected int rotation = 0;
         protected transient Configuration hdfsConfig;
         protected ArrayList<RotationAction> rotationActions = new ArrayList<RotationAction>();
+        protected Object writeLock;
 
         abstract void closeOutputFile() throws IOException;
 
@@ -55,15 +56,17 @@ public class HdfsState implements State {
         protected void rotateOutputFile() throws IOException {
             LOG.info("Rotating output file...");
             long start = System.currentTimeMillis();
-            closeOutputFile();
-            this.rotation++;
-
-            Path newFile = createOutputFile();
-            LOG.info("Performing {} file rotation actions.", this.rotationActions.size());
-            for(RotationAction action : this.rotationActions){
-                action.execute(this.fs, this.currentFile);
+            synchronized (this.writeLock) {
+                closeOutputFile();
+                this.rotation++;
+
+                Path newFile = createOutputFile();
+                LOG.info("Performing {} file rotation actions.", this.rotationActions.size());
+                for (RotationAction action : this.rotationActions) {
+                    action.execute(this.fs, this.currentFile);
+                }
+                this.currentFile = newFile;
             }
-            this.currentFile = newFile;
             long time = System.currentTimeMillis() - start;
             LOG.info("File rotation took {} ms.", time);
 
@@ -71,6 +74,7 @@ public class HdfsState implements State {
         }
 
         void prepare(Map conf, int partitionIndex, int numPartitions){
+            this.writeLock = new Object();
             if (this.rotationPolicy == null) throw new IllegalStateException("RotationPolicy must be specified.");
             if (this.fsUrl == null) {
                 throw new IllegalStateException("File system URL must be specified.");
@@ -152,23 +156,25 @@ public class HdfsState implements State {
         @Override
         public void execute(List<TridentTuple> tuples) throws IOException {
             boolean rotated = false;
-            for(TridentTuple tuple : tuples){
-                byte[] bytes = this.format.format(tuple);
-                out.write(bytes);
-                this.offset += bytes.length;
-
-                if(this.rotationPolicy.mark(tuple, this.offset)){
-                    rotateOutputFile();
-                    this.offset = 0;
-                    this.rotationPolicy.reset();
-                    rotated = true;
+            synchronized (this.writeLock) {
+                for (TridentTuple tuple : tuples) {
+                    byte[] bytes = this.format.format(tuple);
+                    out.write(bytes);
+                    this.offset += bytes.length;
+
+                    if (this.rotationPolicy.mark(tuple, this.offset)) {
+                        rotateOutputFile();
+                        this.offset = 0;
+                        this.rotationPolicy.reset();
+                        rotated = true;
+                    }
                 }
-            }
-            if(!rotated){
-                if(this.out instanceof HdfsDataOutputStream){
-                    ((HdfsDataOutputStream)this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
-                } else {
-                    this.out.hsync();
+                if (!rotated) {
+                    if (this.out instanceof HdfsDataOutputStream) {
+                        ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
+                    } else {
+                        this.out.hsync();
+                    }
                 }
             }
         }
@@ -250,9 +256,12 @@ public class HdfsState implements State {
 
         @Override
         public void execute(List<TridentTuple> tuples) throws IOException {
+            long offset;
             for(TridentTuple tuple : tuples) {
-                this.writer.append(this.format.key(tuple), this.format.value(tuple));
-                long offset = this.writer.getLength();
+                synchronized (this.writeLock) {
+                    this.writer.append(this.format.key(tuple), this.format.value(tuple));
+                    offset = this.writer.getLength();
+                }
 
                 if (this.rotationPolicy.mark(tuple, offset)) {
                     rotateOutputFile();

http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/f227830f/src/test/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java b/src/test/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
index 492839f..48c629e 100644
--- a/src/test/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
+++ b/src/test/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
@@ -98,7 +98,7 @@ public class HdfsFileTopology {
         builder.setBolt(BOLT_ID, bolt, 4)
                 .shuffleGrouping(SENTENCE_SPOUT_ID);
 
-        if (args.length == 1) {
+        if (args.length == 2) {
             LocalCluster cluster = new LocalCluster();
 
             cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());
@@ -106,7 +106,7 @@ public class HdfsFileTopology {
             cluster.killTopology(TOPOLOGY_NAME);
             cluster.shutdown();
             System.exit(0);
-        } else if (args.length == 2) {
+        } else if (args.length == 3) {
             StormSubmitter.submitTopology(args[0], config, builder.createTopology());
         } else{
             System.out.println("Usage: HdfsFileTopology [topology name] <yaml config file>");