You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by pt...@apache.org on 2014/08/08 21:48:31 UTC
[20/50] [abbrv] git commit: implement locking for file writes/rotation
implement locking for file writes/rotation
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/f227830f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/f227830f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/f227830f
Branch: refs/heads/master
Commit: f227830fdbe14290c6870835b3aab6f701dbdddb
Parents: 3509e24
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Thu Jun 26 16:06:30 2014 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Thu Jun 26 16:06:50 2014 -0400
----------------------------------------------------------------------
.../storm/hdfs/bolt/AbstractHdfsBolt.java | 20 ++++---
.../org/apache/storm/hdfs/bolt/HdfsBolt.java | 22 +++----
.../storm/hdfs/bolt/SequenceFileBolt.java | 20 ++++---
.../apache/storm/hdfs/trident/HdfsState.java | 61 +++++++++++---------
.../storm/hdfs/bolt/HdfsFileTopology.java | 4 +-
5 files changed, 73 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/f227830f/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java b/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
index 8d696f4..837b965 100644
--- a/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
+++ b/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
@@ -49,21 +49,24 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
protected int rotation = 0;
protected String fsUrl;
protected String configKey;
+ protected Object writeLock;
protected transient Configuration hdfsConfig;
protected void rotateOutputFile() throws IOException {
LOG.info("Rotating output file...");
long start = System.currentTimeMillis();
- closeOutputFile();
- this.rotation++;
-
- Path newFile = createOutputFile();
- LOG.info("Performing {} file rotation actions.", this.rotationActions.size());
- for(RotationAction action : this.rotationActions){
- action.execute(this.fs, this.currentFile);
+ synchronized (this.writeLock) {
+ closeOutputFile();
+ this.rotation++;
+
+ Path newFile = createOutputFile();
+ LOG.info("Performing {} file rotation actions.", this.rotationActions.size());
+ for (RotationAction action : this.rotationActions) {
+ action.execute(this.fs, this.currentFile);
+ }
+ this.currentFile = newFile;
}
- this.currentFile = newFile;
long time = System.currentTimeMillis() - start;
LOG.info("File rotation took {} ms.", time);
}
@@ -75,6 +78,7 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
* @param collector
*/
public final void prepare(Map conf, TopologyContext topologyContext, OutputCollector collector){
+ this.writeLock = new Object();
if (this.syncPolicy == null) throw new IllegalStateException("SyncPolicy must be specified.");
if (this.rotationPolicy == null) throw new IllegalStateException("RotationPolicy must be specified.");
if (this.fsUrl == null) {
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/f227830f/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java b/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java
index 5c36c65..a416357 100644
--- a/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java
+++ b/src/main/java/org/apache/storm/hdfs/bolt/HdfsBolt.java
@@ -90,22 +90,24 @@ public class HdfsBolt extends AbstractHdfsBolt{
public void execute(Tuple tuple) {
try {
byte[] bytes = this.format.format(tuple);
- out.write(bytes);
- this.offset += bytes.length;
-
- if(this.syncPolicy.mark(tuple, this.offset)){
- if(this.out instanceof HdfsDataOutputStream){
- ((HdfsDataOutputStream)this.out).hsync(EnumSet.of(SyncFlag.UPDATE_LENGTH));
- } else {
- this.out.hsync();
+ synchronized (this.writeLock) {
+ out.write(bytes);
+ this.offset += bytes.length;
+
+ if (this.syncPolicy.mark(tuple, this.offset)) {
+ if (this.out instanceof HdfsDataOutputStream) {
+ ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(SyncFlag.UPDATE_LENGTH));
+ } else {
+ this.out.hsync();
+ }
+ this.syncPolicy.reset();
}
- this.syncPolicy.reset();
}
this.collector.ack(tuple);
if(this.rotationPolicy.mark(tuple, this.offset)){
- rotateOutputFile();
+ rotateOutputFile(); // synchronized
this.offset = 0;
this.rotationPolicy.reset();
}
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/f227830f/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java b/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java
index 6bc5dd2..fc9bb4f 100644
--- a/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java
+++ b/src/main/java/org/apache/storm/hdfs/bolt/SequenceFileBolt.java
@@ -106,16 +106,20 @@ public class SequenceFileBolt extends AbstractHdfsBolt {
@Override
public void execute(Tuple tuple) {
try {
- this.writer.append(this.format.key(tuple), this.format.value(tuple));
- long offset = this.writer.getLength();
- this.collector.ack(tuple);
-
- if (this.syncPolicy.mark(tuple, offset)) {
- this.writer.hsync();
- this.syncPolicy.reset();
+ long offset;
+ synchronized (this.writeLock) {
+ this.writer.append(this.format.key(tuple), this.format.value(tuple));
+ offset = this.writer.getLength();
+
+ if (this.syncPolicy.mark(tuple, offset)) {
+ this.writer.hsync();
+ this.syncPolicy.reset();
+ }
}
+
+ this.collector.ack(tuple);
if (this.rotationPolicy.mark(tuple, offset)) {
- rotateOutputFile();
+ rotateOutputFile(); // synchronized
this.rotationPolicy.reset();
}
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/f227830f/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java b/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
index 01793b9..1c0617a 100644
--- a/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
+++ b/src/main/java/org/apache/storm/hdfs/trident/HdfsState.java
@@ -43,6 +43,7 @@ public class HdfsState implements State {
protected int rotation = 0;
protected transient Configuration hdfsConfig;
protected ArrayList<RotationAction> rotationActions = new ArrayList<RotationAction>();
+ protected Object writeLock;
abstract void closeOutputFile() throws IOException;
@@ -55,15 +56,17 @@ public class HdfsState implements State {
protected void rotateOutputFile() throws IOException {
LOG.info("Rotating output file...");
long start = System.currentTimeMillis();
- closeOutputFile();
- this.rotation++;
-
- Path newFile = createOutputFile();
- LOG.info("Performing {} file rotation actions.", this.rotationActions.size());
- for(RotationAction action : this.rotationActions){
- action.execute(this.fs, this.currentFile);
+ synchronized (this.writeLock) {
+ closeOutputFile();
+ this.rotation++;
+
+ Path newFile = createOutputFile();
+ LOG.info("Performing {} file rotation actions.", this.rotationActions.size());
+ for (RotationAction action : this.rotationActions) {
+ action.execute(this.fs, this.currentFile);
+ }
+ this.currentFile = newFile;
}
- this.currentFile = newFile;
long time = System.currentTimeMillis() - start;
LOG.info("File rotation took {} ms.", time);
@@ -71,6 +74,7 @@ public class HdfsState implements State {
}
void prepare(Map conf, int partitionIndex, int numPartitions){
+ this.writeLock = new Object();
if (this.rotationPolicy == null) throw new IllegalStateException("RotationPolicy must be specified.");
if (this.fsUrl == null) {
throw new IllegalStateException("File system URL must be specified.");
@@ -152,23 +156,25 @@ public class HdfsState implements State {
@Override
public void execute(List<TridentTuple> tuples) throws IOException {
boolean rotated = false;
- for(TridentTuple tuple : tuples){
- byte[] bytes = this.format.format(tuple);
- out.write(bytes);
- this.offset += bytes.length;
-
- if(this.rotationPolicy.mark(tuple, this.offset)){
- rotateOutputFile();
- this.offset = 0;
- this.rotationPolicy.reset();
- rotated = true;
+ synchronized (this.writeLock) {
+ for (TridentTuple tuple : tuples) {
+ byte[] bytes = this.format.format(tuple);
+ out.write(bytes);
+ this.offset += bytes.length;
+
+ if (this.rotationPolicy.mark(tuple, this.offset)) {
+ rotateOutputFile();
+ this.offset = 0;
+ this.rotationPolicy.reset();
+ rotated = true;
+ }
}
- }
- if(!rotated){
- if(this.out instanceof HdfsDataOutputStream){
- ((HdfsDataOutputStream)this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
- } else {
- this.out.hsync();
+ if (!rotated) {
+ if (this.out instanceof HdfsDataOutputStream) {
+ ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
+ } else {
+ this.out.hsync();
+ }
}
}
}
@@ -250,9 +256,12 @@ public class HdfsState implements State {
@Override
public void execute(List<TridentTuple> tuples) throws IOException {
+ long offset;
for(TridentTuple tuple : tuples) {
- this.writer.append(this.format.key(tuple), this.format.value(tuple));
- long offset = this.writer.getLength();
+ synchronized (this.writeLock) {
+ this.writer.append(this.format.key(tuple), this.format.value(tuple));
+ offset = this.writer.getLength();
+ }
if (this.rotationPolicy.mark(tuple, offset)) {
rotateOutputFile();
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/f227830f/src/test/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java b/src/test/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
index 492839f..48c629e 100644
--- a/src/test/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
+++ b/src/test/java/org/apache/storm/hdfs/bolt/HdfsFileTopology.java
@@ -98,7 +98,7 @@ public class HdfsFileTopology {
builder.setBolt(BOLT_ID, bolt, 4)
.shuffleGrouping(SENTENCE_SPOUT_ID);
- if (args.length == 1) {
+ if (args.length == 2) {
LocalCluster cluster = new LocalCluster();
cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());
@@ -106,7 +106,7 @@ public class HdfsFileTopology {
cluster.killTopology(TOPOLOGY_NAME);
cluster.shutdown();
System.exit(0);
- } else if (args.length == 2) {
+ } else if (args.length == 3) {
StormSubmitter.submitTopology(args[0], config, builder.createTopology());
} else{
System.out.println("Usage: HdfsFileTopology [topology name] <yaml config file>");