You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xk...@apache.org on 2018/04/04 22:45:12 UTC
[35/50] [abbrv] hadoop git commit: HADOOP-15146. Remove
DataOutputByteBuffer. Contributed by BELUGA BEHR.
HADOOP-15146. Remove DataOutputByteBuffer.
Contributed by BELUGA BEHR.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dde15790
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dde15790
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dde15790
Branch: refs/heads/HDFS-12943
Commit: dde1579096aa53f0dbb133bcecb64733c9e5b5d6
Parents: a0bde7d
Author: Steve Loughran <st...@apache.org>
Authored: Mon Apr 2 15:52:40 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Apr 2 15:52:40 2018 +0100
----------------------------------------------------------------------
.../apache/hadoop/io/DataOutputByteBuffer.java | 137 -------------
.../apache/hadoop/io/TestDataByteBuffers.java | 196 ++++++-------------
.../org/apache/hadoop/mapreduce/TestTaskID.java | 21 +-
3 files changed, 74 insertions(+), 280 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dde15790/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DataOutputByteBuffer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DataOutputByteBuffer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DataOutputByteBuffer.java
deleted file mode 100644
index b85f693..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/DataOutputByteBuffer.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.io;
-
-import java.io.DataOutputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.LinkedList;
-
-public class DataOutputByteBuffer extends DataOutputStream {
-
- static class Buffer extends OutputStream {
-
- final byte[] b = new byte[1];
- final boolean direct;
- final List<ByteBuffer> active = new ArrayList<ByteBuffer>();
- final List<ByteBuffer> inactive = new LinkedList<ByteBuffer>();
- int size;
- int length;
- ByteBuffer current;
-
- Buffer(int size, boolean direct) {
- this.direct = direct;
- this.size = size;
- current = direct
- ? ByteBuffer.allocateDirect(size)
- : ByteBuffer.allocate(size);
- }
- @Override
- public void write(int b) {
- this.b[0] = (byte)(b & 0xFF);
- write(this.b);
- }
- @Override
- public void write(byte[] b) {
- write(b, 0, b.length);
- }
- @Override
- public void write(byte[] b, int off, int len) {
- int rem = current.remaining();
- while (len > rem) {
- current.put(b, off, rem);
- length += rem;
- current.flip();
- active.add(current);
- off += rem;
- len -= rem;
- rem = getBuffer(len);
- }
- current.put(b, off, len);
- length += len;
- }
- int getBuffer(int newsize) {
- if (inactive.isEmpty()) {
- size = Math.max(size << 1, newsize);
- current = direct
- ? ByteBuffer.allocateDirect(size)
- : ByteBuffer.allocate(size);
- } else {
- current = inactive.remove(0);
- }
- return current.remaining();
- }
- ByteBuffer[] getData() {
- ByteBuffer[] ret = active.toArray(new ByteBuffer[active.size() + 1]);
- ByteBuffer tmp = current.duplicate();
- tmp.flip();
- ret[ret.length - 1] = tmp.slice();
- return ret;
- }
- int getLength() {
- return length;
- }
- void reset() {
- length = 0;
- current.rewind();
- inactive.add(0, current);
- for (int i = active.size() - 1; i >= 0; --i) {
- ByteBuffer b = active.remove(i);
- b.rewind();
- inactive.add(0, b);
- }
- current = inactive.remove(0);
- }
- }
-
- private final Buffer buffers;
-
- public DataOutputByteBuffer() {
- this(32);
- }
-
- public DataOutputByteBuffer(int size) {
- this(size, false);
- }
-
- public DataOutputByteBuffer(int size, boolean direct) {
- this(new Buffer(size, direct));
- }
-
- private DataOutputByteBuffer(Buffer buffers) {
- super(buffers);
- this.buffers = buffers;
- }
-
- public ByteBuffer[] getData() {
- return buffers.getData();
- }
-
- public int getLength() {
- return buffers.getLength();
- }
-
- public void reset() {
- this.written = 0;
- buffers.reset();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dde15790/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestDataByteBuffers.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestDataByteBuffers.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestDataByteBuffers.java
index 1a9dd3b..d06ebaf 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestDataByteBuffers.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestDataByteBuffers.java
@@ -22,7 +22,6 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.Arrays;
import java.util.Random;
import org.junit.Test;
@@ -30,53 +29,67 @@ import static org.junit.Assert.*;
public class TestDataByteBuffers {
- private static void readJunk(DataInput in, Random r, long seed, int iter)
+ private static final Random RAND = new Random(31L);
+
+ private static void readJunk(DataInput in, int iter)
throws IOException {
- r.setSeed(seed);
+ RAND.setSeed(31L);
for (int i = 0; i < iter; ++i) {
- switch (r.nextInt(7)) {
- case 0:
- assertEquals((byte)(r.nextInt() & 0xFF), in.readByte()); break;
- case 1:
- assertEquals((short)(r.nextInt() & 0xFFFF), in.readShort()); break;
- case 2:
- assertEquals(r.nextInt(), in.readInt()); break;
- case 3:
- assertEquals(r.nextLong(), in.readLong()); break;
- case 4:
- assertEquals(Double.doubleToLongBits(r.nextDouble()),
- Double.doubleToLongBits(in.readDouble())); break;
- case 5:
- assertEquals(Float.floatToIntBits(r.nextFloat()),
- Float.floatToIntBits(in.readFloat())); break;
- case 6:
- int len = r.nextInt(1024);
- byte[] vb = new byte[len];
- r.nextBytes(vb);
- byte[] b = new byte[len];
- in.readFully(b, 0, len);
- assertArrayEquals(vb, b);
- break;
+ switch (RAND.nextInt(7)) {
+ case 0:
+ assertEquals((byte)(RAND.nextInt() & 0xFF), in.readByte()); break;
+ case 1:
+ assertEquals((short)(RAND.nextInt() & 0xFFFF), in.readShort()); break;
+ case 2:
+ assertEquals(RAND.nextInt(), in.readInt()); break;
+ case 3:
+ assertEquals(RAND.nextLong(), in.readLong()); break;
+ case 4:
+ assertEquals(Double.doubleToLongBits(RAND.nextDouble()),
+ Double.doubleToLongBits(in.readDouble()));
+ break;
+ case 5:
+ assertEquals(Float.floatToIntBits(RAND.nextFloat()),
+ Float.floatToIntBits(in.readFloat()));
+ break;
+ case 6:
+ int len = RAND.nextInt(1024);
+ byte[] vb = new byte[len];
+ RAND.nextBytes(vb);
+ byte[] b = new byte[len];
+ in.readFully(b, 0, len);
+ assertArrayEquals(vb, b);
+ break;
+ default:
+ throw new IOException();
}
}
}
- private static void writeJunk(DataOutput out, Random r, long seed, int iter)
- throws IOException {
- r.setSeed(seed);
+ private static void writeJunk(DataOutput out, int iter)
+ throws IOException {
+ RAND.setSeed(31L);
for (int i = 0; i < iter; ++i) {
- switch (r.nextInt(7)) {
- case 0: out.writeByte(r.nextInt()); break;
- case 1: out.writeShort((short)(r.nextInt() & 0xFFFF)); break;
- case 2: out.writeInt(r.nextInt()); break;
- case 3: out.writeLong(r.nextLong()); break;
- case 4: out.writeDouble(r.nextDouble()); break;
- case 5: out.writeFloat(r.nextFloat()); break;
- case 6:
- byte[] b = new byte[r.nextInt(1024)];
- r.nextBytes(b);
- out.write(b);
- break;
+ switch (RAND.nextInt(7)) {
+ case 0:
+ out.writeByte(RAND.nextInt()); break;
+ case 1:
+ out.writeShort((short)(RAND.nextInt() & 0xFFFF)); break;
+ case 2:
+ out.writeInt(RAND.nextInt()); break;
+ case 3:
+ out.writeLong(RAND.nextLong()); break;
+ case 4:
+ out.writeDouble(RAND.nextDouble()); break;
+ case 5:
+ out.writeFloat(RAND.nextFloat()); break;
+ case 6:
+ byte[] b = new byte[RAND.nextInt(1024)];
+ RAND.nextBytes(b);
+ out.write(b);
+ break;
+ default:
+ throw new IOException();
}
}
}
@@ -84,113 +97,26 @@ public class TestDataByteBuffers {
@Test
public void testBaseBuffers() throws IOException {
DataOutputBuffer dob = new DataOutputBuffer();
- Random r = new Random();
- long seed = r.nextLong();
- r.setSeed(seed);
- System.out.println("SEED: " + seed);
- writeJunk(dob, r, seed, 1000);
+ writeJunk(dob, 1000);
DataInputBuffer dib = new DataInputBuffer();
dib.reset(dob.getData(), 0, dob.getLength());
- readJunk(dib, r, seed, 1000);
+ readJunk(dib, 1000);
dob.reset();
- writeJunk(dob, r, seed, 1000);
+ writeJunk(dob, 1000);
dib.reset(dob.getData(), 0, dob.getLength());
- readJunk(dib, r, seed, 1000);
- }
-
- @Test
- public void testByteBuffers() throws IOException {
- DataOutputByteBuffer dob = new DataOutputByteBuffer();
- Random r = new Random();
- long seed = r.nextLong();
- r.setSeed(seed);
- System.out.println("SEED: " + seed);
- writeJunk(dob, r, seed, 1000);
- DataInputByteBuffer dib = new DataInputByteBuffer();
- dib.reset(dob.getData());
- readJunk(dib, r, seed, 1000);
-
- dob.reset();
- writeJunk(dob, r, seed, 1000);
- dib.reset(dob.getData());
- readJunk(dib, r, seed, 1000);
- }
-
- private static byte[] toBytes(ByteBuffer[] bufs, int len) {
- byte[] ret = new byte[len];
- int pos = 0;
- for (int i = 0; i < bufs.length; ++i) {
- int rem = bufs[i].remaining();
- bufs[i].get(ret, pos, rem);
- pos += rem;
- }
- return ret;
- }
-
- @Test
- public void testDataOutputByteBufferCompatibility() throws IOException {
- DataOutputBuffer dob = new DataOutputBuffer();
- DataOutputByteBuffer dobb = new DataOutputByteBuffer();
- Random r = new Random();
- long seed = r.nextLong();
- r.setSeed(seed);
- System.out.println("SEED: " + seed);
- writeJunk(dob, r, seed, 1000);
- writeJunk(dobb, r, seed, 1000);
- byte[] check = toBytes(dobb.getData(), dobb.getLength());
- assertEquals(check.length, dob.getLength());
- assertArrayEquals(check, Arrays.copyOf(dob.getData(), dob.getLength()));
-
- dob.reset();
- dobb.reset();
- writeJunk(dob, r, seed, 3000);
- writeJunk(dobb, r, seed, 3000);
- check = toBytes(dobb.getData(), dobb.getLength());
- assertEquals(check.length, dob.getLength());
- assertArrayEquals(check, Arrays.copyOf(dob.getData(), dob.getLength()));
-
- dob.reset();
- dobb.reset();
- writeJunk(dob, r, seed, 1000);
- writeJunk(dobb, r, seed, 1000);
- check = toBytes(dobb.getData(), dobb.getLength());
- assertEquals("Failed Checking length = " + check.length,
- check.length, dob.getLength());
- assertArrayEquals(check, Arrays.copyOf(dob.getData(), dob.getLength()));
+ readJunk(dib, 1000);
}
@Test
- public void TestDataInputByteBufferCompatibility() throws IOException {
+ public void testDataInputByteBufferCompatibility() throws IOException {
DataOutputBuffer dob = new DataOutputBuffer();
- Random r = new Random();
- long seed = r.nextLong();
- r.setSeed(seed);
- System.out.println("SEED: " + seed);
- writeJunk(dob, r, seed, 1000);
+ writeJunk(dob, 1000);
ByteBuffer buf = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
DataInputByteBuffer dib = new DataInputByteBuffer();
dib.reset(buf);
- readJunk(dib, r, seed, 1000);
- }
-
- @Test
- public void TestDataOutputByteBufferCompatibility() throws IOException {
- DataOutputByteBuffer dob = new DataOutputByteBuffer();
- Random r = new Random();
- long seed = r.nextLong();
- r.setSeed(seed);
- System.out.println("SEED: " + seed);
- writeJunk(dob, r, seed, 1000);
- ByteBuffer buf = ByteBuffer.allocate(dob.getLength());
- for (ByteBuffer b : dob.getData()) {
- buf.put(b);
- }
- buf.flip();
- DataInputBuffer dib = new DataInputBuffer();
- dib.reset(buf.array(), 0, buf.remaining());
- readJunk(dib, r, seed, 1000);
+ readJunk(dib, 1000);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dde15790/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestTaskID.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestTaskID.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestTaskID.java
index 5531074..f8b6c0c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestTaskID.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestTaskID.java
@@ -17,8 +17,11 @@
*/
package org.apache.hadoop.mapreduce;
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.nio.ByteBuffer;
+
import org.apache.hadoop.io.DataInputByteBuffer;
-import org.apache.hadoop.io.DataOutputByteBuffer;
import org.apache.hadoop.io.WritableUtils;
import org.junit.Test;
import static org.junit.Assert.*;
@@ -73,7 +76,7 @@ public class TestTaskID {
* Test of getTaskType method, of class TaskID.
*/
@Test
- public void testGetTaskType_0args() {
+ public void testGetTaskType0args() {
JobID jobId = new JobID("1234", 0);
for (TaskType type : TaskType.values()) {
@@ -253,17 +256,18 @@ public class TestTaskID {
*/
@Test
public void testReadFields() throws Exception {
- DataOutputByteBuffer out = new DataOutputByteBuffer();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(baos);
out.writeInt(0);
out.writeInt(1);
WritableUtils.writeVInt(out, 4);
- out.write(new byte[] { 0x31, 0x32, 0x33, 0x34});
+ out.write(new byte[] {0x31, 0x32, 0x33, 0x34});
WritableUtils.writeEnum(out, TaskType.REDUCE);
DataInputByteBuffer in = new DataInputByteBuffer();
- in.reset(out.getData());
+ in.reset(ByteBuffer.wrap(baos.toByteArray()));
TaskID instance = new TaskID();
@@ -280,14 +284,15 @@ public class TestTaskID {
public void testWrite() throws Exception {
JobID jobId = new JobID("1234", 1);
TaskID taskId = new TaskID(jobId, TaskType.JOB_SETUP, 0);
- DataOutputByteBuffer out = new DataOutputByteBuffer();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(baos);
taskId.write(out);
DataInputByteBuffer in = new DataInputByteBuffer();
byte[] buffer = new byte[4];
- in.reset(out.getData());
+ in.reset(ByteBuffer.wrap(baos.toByteArray()));
assertEquals("The write() method did not write the expected task ID",
0, in.readInt());
@@ -430,7 +435,7 @@ public class TestTaskID {
* Test of getTaskType method, of class TaskID.
*/
@Test
- public void testGetTaskType_char() {
+ public void testGetTaskTypeChar() {
assertEquals("The getTaskType() method did not return the expected type",
TaskType.MAP,
TaskID.getTaskType('m'));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org