You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ek...@apache.org on 2015/11/03 19:56:53 UTC
hive git commit: HIVE-12202 NPE thrown when reading legacy ACID delta
files(Elliot West via Eugene Koifman)
Repository: hive
Updated Branches:
refs/heads/branch-1 ac1111a86 -> 703526bc3
HIVE-12202 NPE thrown when reading legacy ACID delta files(Elliot West via Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/703526bc
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/703526bc
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/703526bc
Branch: refs/heads/branch-1
Commit: 703526bc38d73affd1bf1450831927b5e9e24678
Parents: ac1111a
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Tue Nov 3 10:56:43 2015 -0800
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Tue Nov 3 10:56:43 2015 -0800
----------------------------------------------------------------------
.../hadoop/hive/ql/io/AcidInputFormat.java | 14 ++--
.../hadoop/hive/ql/io/TestAcidInputFormat.java | 88 ++++++++++++++++++++
2 files changed, 93 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/703526bc/ql/src/java/org/apache/hadoop/hive/ql/io/AcidInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidInputFormat.java
index 24506b7..7c7074d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidInputFormat.java
@@ -33,7 +33,6 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
/**
@@ -115,11 +114,14 @@ public interface AcidInputFormat<KEY extends WritableComparable, VALUE>
private List<Integer> stmtIds;
public DeltaMetaData() {
- this(0,0,null);
+ this(0,0,new ArrayList<Integer>());
}
DeltaMetaData(long minTxnId, long maxTxnId, List<Integer> stmtIds) {
this.minTxnId = minTxnId;
this.maxTxnId = maxTxnId;
+ if (stmtIds == null) {
+ throw new IllegalArgumentException("stmtIds == null");
+ }
this.stmtIds = stmtIds;
}
long getMinTxnId() {
@@ -136,9 +138,6 @@ public interface AcidInputFormat<KEY extends WritableComparable, VALUE>
out.writeLong(minTxnId);
out.writeLong(maxTxnId);
out.writeInt(stmtIds.size());
- if(stmtIds == null) {
- return;
- }
for(Integer id : stmtIds) {
out.writeInt(id);
}
@@ -147,11 +146,8 @@ public interface AcidInputFormat<KEY extends WritableComparable, VALUE>
public void readFields(DataInput in) throws IOException {
minTxnId = in.readLong();
maxTxnId = in.readLong();
+ stmtIds.clear();
int numStatements = in.readInt();
- if(numStatements <= 0) {
- return;
- }
- stmtIds = new ArrayList<>();
for(int i = 0; i < numStatements; i++) {
stmtIds.add(in.readInt());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/703526bc/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidInputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidInputFormat.java
new file mode 100644
index 0000000..6a77670
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidInputFormat.java
@@ -0,0 +1,88 @@
+package org.apache.hadoop.hive.ql.io;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.DataInput;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.io.AcidInputFormat.DeltaMetaData;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class TestAcidInputFormat {
+
+ @Mock
+ private DataInput mockDataInput;
+
+ @Test
+ public void testDeltaMetaDataReadFieldsNoStatementIds() throws Exception {
+ when(mockDataInput.readLong()).thenReturn(1L, 2L);
+ when(mockDataInput.readInt()).thenReturn(0);
+
+ DeltaMetaData deltaMetaData = new AcidInputFormat.DeltaMetaData();
+ deltaMetaData.readFields(mockDataInput);
+
+ verify(mockDataInput, times(1)).readInt();
+ assertThat(deltaMetaData.getMinTxnId(), is(1L));
+ assertThat(deltaMetaData.getMaxTxnId(), is(2L));
+ assertThat(deltaMetaData.getStmtIds().isEmpty(), is(true));
+ }
+
+ @Test
+ public void testDeltaMetaDataReadFieldsWithStatementIds() throws Exception {
+ when(mockDataInput.readLong()).thenReturn(1L, 2L);
+ when(mockDataInput.readInt()).thenReturn(2, 100, 101);
+
+ DeltaMetaData deltaMetaData = new AcidInputFormat.DeltaMetaData();
+ deltaMetaData.readFields(mockDataInput);
+
+ verify(mockDataInput, times(3)).readInt();
+ assertThat(deltaMetaData.getMinTxnId(), is(1L));
+ assertThat(deltaMetaData.getMaxTxnId(), is(2L));
+ assertThat(deltaMetaData.getStmtIds().size(), is(2));
+ assertThat(deltaMetaData.getStmtIds().get(0), is(100));
+ assertThat(deltaMetaData.getStmtIds().get(1), is(101));
+ }
+
+ @Test
+ public void testDeltaMetaConstructWithState() throws Exception {
+ DeltaMetaData deltaMetaData = new AcidInputFormat.DeltaMetaData(2000L, 2001L, Arrays.asList(97, 98, 99));
+
+ assertThat(deltaMetaData.getMinTxnId(), is(2000L));
+ assertThat(deltaMetaData.getMaxTxnId(), is(2001L));
+ assertThat(deltaMetaData.getStmtIds().size(), is(3));
+ assertThat(deltaMetaData.getStmtIds().get(0), is(97));
+ assertThat(deltaMetaData.getStmtIds().get(1), is(98));
+ assertThat(deltaMetaData.getStmtIds().get(2), is(99));
+ }
+
+ @Test
+ public void testDeltaMetaDataReadFieldsWithStatementIdsResetsState() throws Exception {
+ when(mockDataInput.readLong()).thenReturn(1L, 2L);
+ when(mockDataInput.readInt()).thenReturn(2, 100, 101);
+
+ List<Integer> statementIds = new ArrayList<>();
+ statementIds.add(97);
+ statementIds.add(98);
+ statementIds.add(99);
+ DeltaMetaData deltaMetaData = new AcidInputFormat.DeltaMetaData(2000L, 2001L, statementIds);
+ deltaMetaData.readFields(mockDataInput);
+
+ verify(mockDataInput, times(3)).readInt();
+ assertThat(deltaMetaData.getMinTxnId(), is(1L));
+ assertThat(deltaMetaData.getMaxTxnId(), is(2L));
+ assertThat(deltaMetaData.getStmtIds().size(), is(2));
+ assertThat(deltaMetaData.getStmtIds().get(0), is(100));
+ assertThat(deltaMetaData.getStmtIds().get(1), is(101));
+ }
+
+}