You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2015/12/16 20:17:36 UTC
[1/3] hadoop git commit: HDFS-9557. Reduce object allocation in PB
conversion. Contributed by Daryn Sharp.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 1a2ef845b -> e8e3737ce
refs/heads/branch-2.8 c4febc7d0 -> d3e1161b8
refs/heads/trunk 50bd067e1 -> c470c8953
HDFS-9557. Reduce object allocation in PB conversion. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c470c895
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c470c895
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c470c895
Branch: refs/heads/trunk
Commit: c470c8953d4927043b6383fad8e792289c634c09
Parents: 50bd067
Author: cnauroth <cn...@apache.org>
Authored: Wed Dec 16 10:45:32 2015 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Wed Dec 16 11:10:40 2015 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 29 ++++++++++----------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 2 +-
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 7 +++++
4 files changed, 26 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c470c895/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 33bdc5c..ac960e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -194,7 +194,8 @@ public class PBHelperClient {
}
public static ByteString getByteString(byte[] bytes) {
- return ByteString.copyFrom(bytes);
+ // return singleton to reduce object allocation
+ return (bytes.length == 0) ? ByteString.EMPTY : ByteString.copyFrom(bytes);
}
public static ShmId convert(ShortCircuitShmIdProto shmId) {
@@ -221,8 +222,8 @@ public class PBHelperClient {
public static TokenProto convert(Token<?> tok) {
return TokenProto.newBuilder().
- setIdentifier(ByteString.copyFrom(tok.getIdentifier())).
- setPassword(ByteString.copyFrom(tok.getPassword())).
+ setIdentifier(getByteString(tok.getIdentifier())).
+ setPassword(getByteString(tok.getPassword())).
setKind(tok.getKind().toString()).
setService(tok.getService().toString()).build();
}
@@ -451,16 +452,16 @@ public class PBHelperClient {
builder.setSuite(convert(option.getCipherSuite()));
}
if (option.getInKey() != null) {
- builder.setInKey(ByteString.copyFrom(option.getInKey()));
+ builder.setInKey(getByteString(option.getInKey()));
}
if (option.getInIv() != null) {
- builder.setInIv(ByteString.copyFrom(option.getInIv()));
+ builder.setInIv(getByteString(option.getInIv()));
}
if (option.getOutKey() != null) {
- builder.setOutKey(ByteString.copyFrom(option.getOutKey()));
+ builder.setOutKey(getByteString(option.getOutKey()));
}
if (option.getOutIv() != null) {
- builder.setOutIv(ByteString.copyFrom(option.getOutIv()));
+ builder.setOutIv(getByteString(option.getOutIv()));
}
return builder.build();
}
@@ -1738,8 +1739,8 @@ public class PBHelperClient {
DataEncryptionKeyProto.Builder b = DataEncryptionKeyProto.newBuilder()
.setKeyId(bet.keyId)
.setBlockPoolId(bet.blockPoolId)
- .setNonce(ByteString.copyFrom(bet.nonce))
- .setEncryptionKey(ByteString.copyFrom(bet.encryptionKey))
+ .setNonce(getByteString(bet.nonce))
+ .setEncryptionKey(getByteString(bet.encryptionKey))
.setExpiryDate(bet.expiryDate);
if (bet.encryptionAlgorithm != null) {
b.setEncryptionAlgorithm(bet.encryptionAlgorithm);
@@ -1816,10 +1817,10 @@ public class PBHelperClient {
setGroup(fs.getGroup()).
setFileId(fs.getFileId()).
setChildrenNum(fs.getChildrenNum()).
- setPath(ByteString.copyFrom(fs.getLocalNameInBytes())).
+ setPath(getByteString(fs.getLocalNameInBytes())).
setStoragePolicy(fs.getStoragePolicy());
if (fs.isSymlink()) {
- builder.setSymlink(ByteString.copyFrom(fs.getSymlinkInBytes()));
+ builder.setSymlink(getByteString(fs.getSymlinkInBytes()));
}
if (fs.getFileEncryptionInfo() != null) {
builder.setFileEncryptionInfo(convert(fs.getFileEncryptionInfo()));
@@ -1846,7 +1847,7 @@ public class PBHelperClient {
int snapshotNumber = status.getSnapshotNumber();
int snapshotQuota = status.getSnapshotQuota();
byte[] parentFullPath = status.getParentFullPath();
- ByteString parentFullPathBytes = ByteString.copyFrom(
+ ByteString parentFullPathBytes = getByteString(
parentFullPath == null ? DFSUtilClient.EMPTY_BYTES : parentFullPath);
HdfsFileStatusProto fs = convert(status.getDirStatus());
SnapshottableDirectoryStatusProto.Builder builder =
@@ -2061,7 +2062,7 @@ public class PBHelperClient {
if (entry == null) {
return null;
}
- ByteString sourcePath = ByteString.copyFrom(entry.getSourcePath() == null ?
+ ByteString sourcePath = getByteString(entry.getSourcePath() == null ?
DFSUtilClient.EMPTY_BYTES : entry.getSourcePath());
String modification = entry.getType().getLabel();
SnapshotDiffReportEntryProto.Builder builder = SnapshotDiffReportEntryProto
@@ -2069,7 +2070,7 @@ public class PBHelperClient {
.setModificationLabel(modification);
if (entry.getType() == DiffType.RENAME) {
ByteString targetPath =
- ByteString.copyFrom(entry.getTargetPath() == null ?
+ getByteString(entry.getTargetPath() == null ?
DFSUtilClient.EMPTY_BYTES : entry.getTargetPath());
builder.setTargetPath(targetPath);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c470c895/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index c11b48f..69fa302 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1764,6 +1764,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8894. Set SO_KEEPALIVE on DN server sockets.
(Kanaka Kumar Avvaru via wang)
+ HDFS-9557. Reduce object allocation in PB conversion
+ (Daryn Sharp via cnauroth)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c470c895/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index fd8a386..9271e33 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -229,7 +229,7 @@ public class PBHelper {
public static BlockKeyProto convert(BlockKey key) {
byte[] encodedKey = key.getEncodedKey();
- ByteString keyBytes = ByteString.copyFrom(encodedKey == null ?
+ ByteString keyBytes = PBHelperClient.getByteString(encodedKey == null ?
DFSUtilClient.EMPTY_BYTES : encodedKey);
return BlockKeyProto.newBuilder().setKeyId(key.getKeyId())
.setKeyBytes(keyBytes).setExpiryDate(key.getExpiryDate()).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c470c895/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
index 4171d5c..9f0e0f1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.protocolPB;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@@ -99,6 +100,7 @@ import org.junit.Test;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import com.google.protobuf.ByteString;
/**
* Tests for {@link PBHelper}
@@ -111,6 +113,11 @@ public class TestPBHelper {
private static final double DELTA = 0.000001;
@Test
+ public void testGetByteString() {
+ assertSame(ByteString.EMPTY, PBHelperClient.getByteString(new byte[0]));
+ }
+
+ @Test
public void testConvertNamenodeRole() {
assertEquals(NamenodeRoleProto.BACKUP,
PBHelper.convert(NamenodeRole.BACKUP));
[3/3] hadoop git commit: HDFS-9557. Reduce object allocation in PB
conversion. Contributed by Daryn Sharp.
Posted by cn...@apache.org.
HDFS-9557. Reduce object allocation in PB conversion. Contributed by Daryn Sharp.
(cherry picked from commit c470c8953d4927043b6383fad8e792289c634c09)
(cherry picked from commit e8e3737ce8224df41f57b5d03ed097e8ab413bda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3e1161b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3e1161b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3e1161b
Branch: refs/heads/branch-2.8
Commit: d3e1161b8d7128136402a71e7b4dfb87b6377d2c
Parents: c4febc7
Author: cnauroth <cn...@apache.org>
Authored: Wed Dec 16 10:45:32 2015 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Wed Dec 16 11:11:34 2015 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 29 ++++++++++----------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 2 +-
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 7 +++++
4 files changed, 26 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3e1161b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index f35f6a5..d64bdd9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -189,7 +189,8 @@ public class PBHelperClient {
}
public static ByteString getByteString(byte[] bytes) {
- return ByteString.copyFrom(bytes);
+ // return singleton to reduce object allocation
+ return (bytes.length == 0) ? ByteString.EMPTY : ByteString.copyFrom(bytes);
}
public static ShmId convert(ShortCircuitShmIdProto shmId) {
@@ -216,8 +217,8 @@ public class PBHelperClient {
public static TokenProto convert(Token<?> tok) {
return TokenProto.newBuilder().
- setIdentifier(ByteString.copyFrom(tok.getIdentifier())).
- setPassword(ByteString.copyFrom(tok.getPassword())).
+ setIdentifier(getByteString(tok.getIdentifier())).
+ setPassword(getByteString(tok.getPassword())).
setKind(tok.getKind().toString()).
setService(tok.getService().toString()).build();
}
@@ -446,16 +447,16 @@ public class PBHelperClient {
builder.setSuite(convert(option.getCipherSuite()));
}
if (option.getInKey() != null) {
- builder.setInKey(ByteString.copyFrom(option.getInKey()));
+ builder.setInKey(getByteString(option.getInKey()));
}
if (option.getInIv() != null) {
- builder.setInIv(ByteString.copyFrom(option.getInIv()));
+ builder.setInIv(getByteString(option.getInIv()));
}
if (option.getOutKey() != null) {
- builder.setOutKey(ByteString.copyFrom(option.getOutKey()));
+ builder.setOutKey(getByteString(option.getOutKey()));
}
if (option.getOutIv() != null) {
- builder.setOutIv(ByteString.copyFrom(option.getOutIv()));
+ builder.setOutIv(getByteString(option.getOutIv()));
}
return builder.build();
}
@@ -1676,8 +1677,8 @@ public class PBHelperClient {
DataEncryptionKeyProto.Builder b = DataEncryptionKeyProto.newBuilder()
.setKeyId(bet.keyId)
.setBlockPoolId(bet.blockPoolId)
- .setNonce(ByteString.copyFrom(bet.nonce))
- .setEncryptionKey(ByteString.copyFrom(bet.encryptionKey))
+ .setNonce(getByteString(bet.nonce))
+ .setEncryptionKey(getByteString(bet.encryptionKey))
.setExpiryDate(bet.expiryDate);
if (bet.encryptionAlgorithm != null) {
b.setEncryptionAlgorithm(bet.encryptionAlgorithm);
@@ -1754,10 +1755,10 @@ public class PBHelperClient {
setGroup(fs.getGroup()).
setFileId(fs.getFileId()).
setChildrenNum(fs.getChildrenNum()).
- setPath(ByteString.copyFrom(fs.getLocalNameInBytes())).
+ setPath(getByteString(fs.getLocalNameInBytes())).
setStoragePolicy(fs.getStoragePolicy());
if (fs.isSymlink()) {
- builder.setSymlink(ByteString.copyFrom(fs.getSymlinkInBytes()));
+ builder.setSymlink(getByteString(fs.getSymlinkInBytes()));
}
if (fs.getFileEncryptionInfo() != null) {
builder.setFileEncryptionInfo(convert(fs.getFileEncryptionInfo()));
@@ -1780,7 +1781,7 @@ public class PBHelperClient {
int snapshotNumber = status.getSnapshotNumber();
int snapshotQuota = status.getSnapshotQuota();
byte[] parentFullPath = status.getParentFullPath();
- ByteString parentFullPathBytes = ByteString.copyFrom(
+ ByteString parentFullPathBytes = getByteString(
parentFullPath == null ? DFSUtilClient.EMPTY_BYTES : parentFullPath);
HdfsFileStatusProto fs = convert(status.getDirStatus());
SnapshottableDirectoryStatusProto.Builder builder =
@@ -1995,7 +1996,7 @@ public class PBHelperClient {
if (entry == null) {
return null;
}
- ByteString sourcePath = ByteString.copyFrom(entry.getSourcePath() == null ?
+ ByteString sourcePath = getByteString(entry.getSourcePath() == null ?
DFSUtilClient.EMPTY_BYTES : entry.getSourcePath());
String modification = entry.getType().getLabel();
SnapshotDiffReportEntryProto.Builder builder = SnapshotDiffReportEntryProto
@@ -2003,7 +2004,7 @@ public class PBHelperClient {
.setModificationLabel(modification);
if (entry.getType() == DiffType.RENAME) {
ByteString targetPath =
- ByteString.copyFrom(entry.getTargetPath() == null ?
+ getByteString(entry.getTargetPath() == null ?
DFSUtilClient.EMPTY_BYTES : entry.getTargetPath());
builder.setTargetPath(targetPath);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3e1161b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index aee4def..994f28e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -859,6 +859,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8894. Set SO_KEEPALIVE on DN server sockets.
(Kanaka Kumar Avvaru via wang)
+ HDFS-9557. Reduce object allocation in PB conversion
+ (Daryn Sharp via cnauroth)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3e1161b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index a3bf0de..dab62e3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -209,7 +209,7 @@ public class PBHelper {
public static BlockKeyProto convert(BlockKey key) {
byte[] encodedKey = key.getEncodedKey();
- ByteString keyBytes = ByteString.copyFrom(encodedKey == null ?
+ ByteString keyBytes = PBHelperClient.getByteString(encodedKey == null ?
DFSUtilClient.EMPTY_BYTES : encodedKey);
return BlockKeyProto.newBuilder().setKeyId(key.getKeyId())
.setKeyBytes(keyBytes).setExpiryDate(key.getExpiryDate()).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3e1161b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
index 2bfba98..aa93328 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.protocolPB;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@@ -89,6 +90,7 @@ import org.junit.Test;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import com.google.protobuf.ByteString;
/**
* Tests for {@link PBHelper}
@@ -101,6 +103,11 @@ public class TestPBHelper {
private static final double DELTA = 0.000001;
@Test
+ public void testGetByteString() {
+ assertSame(ByteString.EMPTY, PBHelperClient.getByteString(new byte[0]));
+ }
+
+ @Test
public void testConvertNamenodeRole() {
assertEquals(NamenodeRoleProto.BACKUP,
PBHelper.convert(NamenodeRole.BACKUP));
[2/3] hadoop git commit: HDFS-9557. Reduce object allocation in PB
conversion. Contributed by Daryn Sharp.
Posted by cn...@apache.org.
HDFS-9557. Reduce object allocation in PB conversion. Contributed by Daryn Sharp.
(cherry picked from commit c470c8953d4927043b6383fad8e792289c634c09)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8e3737c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8e3737c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8e3737c
Branch: refs/heads/branch-2
Commit: e8e3737ce8224df41f57b5d03ed097e8ab413bda
Parents: 1a2ef84
Author: cnauroth <cn...@apache.org>
Authored: Wed Dec 16 10:45:32 2015 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Wed Dec 16 11:11:27 2015 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 29 ++++++++++----------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 2 +-
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 7 +++++
4 files changed, 26 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8e3737c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index f35f6a5..d64bdd9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -189,7 +189,8 @@ public class PBHelperClient {
}
public static ByteString getByteString(byte[] bytes) {
- return ByteString.copyFrom(bytes);
+ // return singleton to reduce object allocation
+ return (bytes.length == 0) ? ByteString.EMPTY : ByteString.copyFrom(bytes);
}
public static ShmId convert(ShortCircuitShmIdProto shmId) {
@@ -216,8 +217,8 @@ public class PBHelperClient {
public static TokenProto convert(Token<?> tok) {
return TokenProto.newBuilder().
- setIdentifier(ByteString.copyFrom(tok.getIdentifier())).
- setPassword(ByteString.copyFrom(tok.getPassword())).
+ setIdentifier(getByteString(tok.getIdentifier())).
+ setPassword(getByteString(tok.getPassword())).
setKind(tok.getKind().toString()).
setService(tok.getService().toString()).build();
}
@@ -446,16 +447,16 @@ public class PBHelperClient {
builder.setSuite(convert(option.getCipherSuite()));
}
if (option.getInKey() != null) {
- builder.setInKey(ByteString.copyFrom(option.getInKey()));
+ builder.setInKey(getByteString(option.getInKey()));
}
if (option.getInIv() != null) {
- builder.setInIv(ByteString.copyFrom(option.getInIv()));
+ builder.setInIv(getByteString(option.getInIv()));
}
if (option.getOutKey() != null) {
- builder.setOutKey(ByteString.copyFrom(option.getOutKey()));
+ builder.setOutKey(getByteString(option.getOutKey()));
}
if (option.getOutIv() != null) {
- builder.setOutIv(ByteString.copyFrom(option.getOutIv()));
+ builder.setOutIv(getByteString(option.getOutIv()));
}
return builder.build();
}
@@ -1676,8 +1677,8 @@ public class PBHelperClient {
DataEncryptionKeyProto.Builder b = DataEncryptionKeyProto.newBuilder()
.setKeyId(bet.keyId)
.setBlockPoolId(bet.blockPoolId)
- .setNonce(ByteString.copyFrom(bet.nonce))
- .setEncryptionKey(ByteString.copyFrom(bet.encryptionKey))
+ .setNonce(getByteString(bet.nonce))
+ .setEncryptionKey(getByteString(bet.encryptionKey))
.setExpiryDate(bet.expiryDate);
if (bet.encryptionAlgorithm != null) {
b.setEncryptionAlgorithm(bet.encryptionAlgorithm);
@@ -1754,10 +1755,10 @@ public class PBHelperClient {
setGroup(fs.getGroup()).
setFileId(fs.getFileId()).
setChildrenNum(fs.getChildrenNum()).
- setPath(ByteString.copyFrom(fs.getLocalNameInBytes())).
+ setPath(getByteString(fs.getLocalNameInBytes())).
setStoragePolicy(fs.getStoragePolicy());
if (fs.isSymlink()) {
- builder.setSymlink(ByteString.copyFrom(fs.getSymlinkInBytes()));
+ builder.setSymlink(getByteString(fs.getSymlinkInBytes()));
}
if (fs.getFileEncryptionInfo() != null) {
builder.setFileEncryptionInfo(convert(fs.getFileEncryptionInfo()));
@@ -1780,7 +1781,7 @@ public class PBHelperClient {
int snapshotNumber = status.getSnapshotNumber();
int snapshotQuota = status.getSnapshotQuota();
byte[] parentFullPath = status.getParentFullPath();
- ByteString parentFullPathBytes = ByteString.copyFrom(
+ ByteString parentFullPathBytes = getByteString(
parentFullPath == null ? DFSUtilClient.EMPTY_BYTES : parentFullPath);
HdfsFileStatusProto fs = convert(status.getDirStatus());
SnapshottableDirectoryStatusProto.Builder builder =
@@ -1995,7 +1996,7 @@ public class PBHelperClient {
if (entry == null) {
return null;
}
- ByteString sourcePath = ByteString.copyFrom(entry.getSourcePath() == null ?
+ ByteString sourcePath = getByteString(entry.getSourcePath() == null ?
DFSUtilClient.EMPTY_BYTES : entry.getSourcePath());
String modification = entry.getType().getLabel();
SnapshotDiffReportEntryProto.Builder builder = SnapshotDiffReportEntryProto
@@ -2003,7 +2004,7 @@ public class PBHelperClient {
.setModificationLabel(modification);
if (entry.getType() == DiffType.RENAME) {
ByteString targetPath =
- ByteString.copyFrom(entry.getTargetPath() == null ?
+ getByteString(entry.getTargetPath() == null ?
DFSUtilClient.EMPTY_BYTES : entry.getTargetPath());
builder.setTargetPath(targetPath);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8e3737c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1395b4d..00b4547 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -898,6 +898,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8894. Set SO_KEEPALIVE on DN server sockets.
(Kanaka Kumar Avvaru via wang)
+ HDFS-9557. Reduce object allocation in PB conversion
+ (Daryn Sharp via cnauroth)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8e3737c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index a3bf0de..dab62e3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -209,7 +209,7 @@ public class PBHelper {
public static BlockKeyProto convert(BlockKey key) {
byte[] encodedKey = key.getEncodedKey();
- ByteString keyBytes = ByteString.copyFrom(encodedKey == null ?
+ ByteString keyBytes = PBHelperClient.getByteString(encodedKey == null ?
DFSUtilClient.EMPTY_BYTES : encodedKey);
return BlockKeyProto.newBuilder().setKeyId(key.getKeyId())
.setKeyBytes(keyBytes).setExpiryDate(key.getExpiryDate()).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8e3737c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
index 2bfba98..aa93328 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.protocolPB;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@@ -89,6 +90,7 @@ import org.junit.Test;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import com.google.protobuf.ByteString;
/**
* Tests for {@link PBHelper}
@@ -101,6 +103,11 @@ public class TestPBHelper {
private static final double DELTA = 0.000001;
@Test
+ public void testGetByteString() {
+ assertSame(ByteString.EMPTY, PBHelperClient.getByteString(new byte[0]));
+ }
+
+ @Test
public void testConvertNamenodeRole() {
assertEquals(NamenodeRoleProto.BACKUP,
PBHelper.convert(NamenodeRole.BACKUP));