You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by li...@apache.org on 2015/07/13 06:46:51 UTC
hadoop git commit: HDFS-8744. Erasure Coding: the number of chunks in
packet is not updated when writing parity data. Contributed by Li Bo
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285 e692c7dd9 -> f4098dfae
HDFS-8744. Erasure Coding: the number of chunks in packet is not updated when writing parity data. Contributed by Li Bo
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4098dfa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4098dfa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4098dfa
Branch: refs/heads/HDFS-7285
Commit: f4098dfae44b6dd287a3054f2c04658773b4f466
Parents: e692c7d
Author: boli2 <bo...@intel.com>
Authored: Mon Jul 13 00:41:36 2015 -0400
Committer: boli2 <bo...@intel.com>
Committed: Mon Jul 13 00:41:36 2015 -0400
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 +++
.../src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java | 2 +-
.../src/main/java/org/apache/hadoop/hdfs/DFSPacket.java | 6 +++---
.../java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java | 4 +++-
4 files changed, 10 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4098dfa/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index 90f6732..7b6d165 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -341,3 +341,6 @@
HDFS-8484. Erasure coding: Two contiguous blocks occupy IDs belong to same
striped group. (Walter Su via jing9)
+
+ HDFS-8744. Erasure Coding: the number of chunks in packet is not updated
+ when writing parity data. (Li Bo)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4098dfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index 9e201ad..f41044b 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -419,7 +419,7 @@ public class DFSOutputStream extends FSOutputSummer
currentPacket.writeChecksum(checksum, ckoff, cklen);
currentPacket.writeData(b, offset, len);
- currentPacket.incNumChunks();
+ currentPacket.incNumChunks(1);
streamer.incBytesCurBlock(len);
// If packet is full, enqueue it for transmission
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4098dfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
index a26e35e..2698de3 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
@@ -259,10 +259,10 @@ public class DFSPacket {
}
/**
- * increase the number of chunks by one
+ * increase the number of chunks by n
*/
- synchronized void incNumChunks() {
- numChunks++;
+ synchronized void incNumChunks(int n) {
+ numChunks += n;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4098dfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index 4234351..e6de714 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -389,11 +389,13 @@ public class DFSStripedOutputStream extends DFSOutputStream {
int maxBytesToPacket = p.getMaxChunks() * bytesPerChecksum;
int toWrite = byteBuffer.remaining() > maxBytesToPacket ?
maxBytesToPacket: byteBuffer.remaining();
- int ckLen = ((toWrite - 1) / bytesPerChecksum + 1) * getChecksumSize();
+ int chunks = (toWrite - 1) / bytesPerChecksum + 1;
+ int ckLen = chunks * getChecksumSize();
p.writeChecksum(checksumBuf, ckOff, ckLen);
ckOff += ckLen;
p.writeData(byteBuffer, toWrite);
getCurrentStreamer().incBytesCurBlock(toWrite);
+ p.incNumChunks(chunks);
packets.add(p);
}
return packets;