You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mu...@apache.org on 2020/11/20 19:08:29 UTC
[hive] branch branch-3.1 updated: HIVE-24391: Fix FIX TestOrcFile
failures in branch-3.1 (Panagiotis Garefalakis via Mustafa Iman)
This is an automated email from the ASF dual-hosted git repository.
mustafaiman pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new c3ad198 HIVE-24391: Fix FIX TestOrcFile failures in branch-3.1 (Panagiotis Garefalakis via Mustafa Iman)
c3ad198 is described below
commit c3ad198ef026446a0c2bf086a0869b98c66fad51
Author: Panagiotis Garefalakis <pg...@cloudera.com>
AuthorDate: Sat Nov 30 21:51:35 2019 +0000
HIVE-24391: Fix FIX TestOrcFile failures in branch-3.1 (Panagiotis Garefalakis via Mustafa Iman)
backport HIVE-22555
Change-Id: I5a2e684000040f47c380e20842ce5e15b3d68669
---
.../apache/hadoop/hive/ql/io/orc/TestOrcFile.java | 82 ++++++++--------------
.../hive/ql/io/orc/TestOrcRawRecordMerger.java | 40 ++---------
2 files changed, 36 insertions(+), 86 deletions(-)
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
index 97d4fc6..a546ffb 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
@@ -91,6 +91,7 @@ import org.apache.orc.StringColumnStatistics;
import org.apache.orc.StripeInformation;
import org.apache.orc.StripeStatistics;
import org.apache.orc.TypeDescription;
+import org.apache.orc.impl.MemoryManagerImpl;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
@@ -1922,52 +1923,22 @@ public class TestOrcFile {
new MiddleStruct(inner, inner2), list(), map(inner,inner2));
}
- private static class MyMemoryManager implements MemoryManager {
- final long totalSpace;
- double rate;
- Path path = null;
- long lastAllocation = 0;
- int rows = 0;
- MemoryManager.Callback callback;
-
- MyMemoryManager(Configuration conf, long totalSpace, double rate) {
- this.totalSpace = totalSpace;
- this.rate = rate;
- }
-
- @Override
- public void addWriter(Path path, long requestedAllocation,
- MemoryManager.Callback callback) {
- this.path = path;
- this.lastAllocation = requestedAllocation;
- this.callback = callback;
- }
-
- @Override
- public synchronized void removeWriter(Path path) {
- this.path = null;
- this.lastAllocation = 0;
- }
-
- @Override
- public void addedRow(int count) throws IOException {
- rows += count;
- if (rows >= 100) {
- callback.checkMemory(rate);
- rows = 0;
- }
- }
- }
-
@Test
public void testMemoryManagementV11() throws Exception {
+ OrcConf.ROWS_BETWEEN_CHECKS.setLong(conf, 100);
+ final long poolSize = 50_000;
ObjectInspector inspector;
synchronized (TestOrcFile.class) {
inspector = ObjectInspectorFactory.getReflectionObjectInspector
(InnerStruct.class,
ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
}
- MyMemoryManager memory = new MyMemoryManager(conf, 10000, 0.1);
+ MemoryManager memoryManager = new MemoryManagerImpl(poolSize);
+ // set up 10 files that all request the full size.
+ MemoryManager.Callback ignore = newScale -> false;
+ for(int f=0; f < 9; ++f) {
+ memoryManager.addWriter(new Path("file-" + f), poolSize, ignore);
+ }
Writer writer = OrcFile.createWriter(testFilePath,
OrcFile.writerOptions(conf)
.inspector(inspector)
@@ -1975,15 +1946,14 @@ public class TestOrcFile {
.stripeSize(50000)
.bufferSize(100)
.rowIndexStride(0)
- .memory(memory)
+ .memory(memoryManager)
.batchSize(100)
.version(OrcFile.Version.V_0_11));
- assertEquals(testFilePath, memory.path);
+ assertEquals(0.1, ((MemoryManagerImpl) memoryManager).getAllocationScale());
for(int i=0; i < 2500; ++i) {
writer.addRow(new InnerStruct(i*300, Integer.toHexString(10*i)));
}
writer.close();
- assertEquals(null, memory.path);
Reader reader = OrcFile.createReader(testFilePath,
OrcFile.readerOptions(conf).filesystem(fs));
int i = 0;
@@ -1998,29 +1968,35 @@ public class TestOrcFile {
@Test
public void testMemoryManagementV12() throws Exception {
+ OrcConf.ROWS_BETWEEN_CHECKS.setLong(conf, 100);
+ final long poolSize = 50_000;
ObjectInspector inspector;
synchronized (TestOrcFile.class) {
inspector = ObjectInspectorFactory.getReflectionObjectInspector
(InnerStruct.class,
ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
}
- MyMemoryManager memory = new MyMemoryManager(conf, 10000, 0.1);
+ MemoryManager memoryManager = new MemoryManagerImpl(poolSize);
+ // set up 10 files that all request the full size.
+ MemoryManager.Callback ignore = newScale -> false;
+ for(int f=0; f < 9; ++f) {
+ memoryManager.addWriter(new Path("file-" + f), poolSize, ignore);
+ }
Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .compress(CompressionKind.NONE)
- .stripeSize(50000)
- .bufferSize(100)
- .rowIndexStride(0)
- .memory(memory)
- .batchSize(100)
- .version(OrcFile.Version.V_0_12));
- assertEquals(testFilePath, memory.path);
+ OrcFile.writerOptions(conf)
+ .inspector(inspector)
+ .compress(CompressionKind.NONE)
+ .stripeSize(50000)
+ .bufferSize(100)
+ .rowIndexStride(0)
+ .memory(memoryManager)
+ .batchSize(100)
+ .version(OrcFile.Version.V_0_12));
+ assertEquals(0.1, ((MemoryManagerImpl) memoryManager).getAllocationScale());
for(int i=0; i < 2500; ++i) {
writer.addRow(new InnerStruct(i*300, Integer.toHexString(10*i)));
}
writer.close();
- assertEquals(null, memory.path);
Reader reader = OrcFile.createReader(testFilePath,
OrcFile.readerOptions(conf).filesystem(fs));
int i = 0;
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
index cc29384..753c707 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
@@ -23,9 +23,8 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.io.BucketCodec;
import org.apache.orc.CompressionKind;
-import org.apache.orc.MemoryManager;
+import org.apache.orc.OrcConf;
import org.apache.orc.StripeInformation;
-import org.apache.orc.impl.MemoryManagerImpl;
import org.apache.orc.impl.OrcAcidUtils;
import org.junit.Assert;
import org.junit.Rule;
@@ -1064,6 +1063,7 @@ public class TestOrcRawRecordMerger {
public void testRecordReaderOldBaseAndDelta() throws Exception {
final int BUCKET = 10;
Configuration conf = new Configuration();
+ OrcConf.ROWS_BETWEEN_CHECKS.setLong(conf, 2);
OrcOutputFormat of = new OrcOutputFormat();
FileSystem fs = FileSystem.getLocal(conf);
Path root = new Path(tmpDir, "testOldBaseAndDelta").makeQualified(fs);
@@ -1073,25 +1073,11 @@ public class TestOrcRawRecordMerger {
inspector = ObjectInspectorFactory.getReflectionObjectInspector
(BigRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
}
-
- // write the base
- MemoryManager mgr = new MemoryManagerImpl(conf){
- int rowsAddedSinceCheck = 0;
-
- @Override
- public synchronized void addedRow(int rows) throws IOException {
- rowsAddedSinceCheck += rows;
- if (rowsAddedSinceCheck >= 2) {
- notifyWriters();
- rowsAddedSinceCheck = 0;
- }
- }
- };
// make 5 stripes with 2 rows each
Writer writer = OrcFile.createWriter(new Path(root, "0000010_0"),
OrcFile.writerOptions(conf).inspector(inspector).fileSystem(fs)
.blockPadding(false).bufferSize(10000).compress(CompressionKind.NONE)
- .stripeSize(1).memory(mgr).batchSize(2).version(OrcFile.Version.V_0_11));
+ .stripeSize(1).batchSize(2).version(OrcFile.Version.V_0_11));
String[] values= new String[]{"ignore.1", "0.1", "ignore.2", "ignore.3",
"2.0", "2.1", "3.0", "ignore.4", "ignore.5", "ignore.6"};
for(int i=0; i < values.length; ++i) {
@@ -1195,6 +1181,7 @@ public class TestOrcRawRecordMerger {
public void testRecordReaderNewBaseAndDelta() throws Exception {
final int BUCKET = 11;
Configuration conf = new Configuration();
+ OrcConf.ROWS_BETWEEN_CHECKS.setLong(conf, 2);
OrcOutputFormat of = new OrcOutputFormat();
FileSystem fs = FileSystem.getLocal(conf);
Path root = new Path(tmpDir, "testRecordReaderNewBaseAndDelta").makeQualified(fs);
@@ -1205,20 +1192,6 @@ public class TestOrcRawRecordMerger {
(BigRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
}
- // write the base
- MemoryManager mgr = new MemoryManagerImpl(conf){
- int rowsAddedSinceCheck = 0;
-
- @Override
- public synchronized void addedRow(int rows) throws IOException {
- rowsAddedSinceCheck += rows;
- if (rowsAddedSinceCheck >= 2) {
- notifyWriters();
- rowsAddedSinceCheck = 0;
- }
- }
- };
-
// make 5 stripes with 2 rows each
OrcRecordUpdater.OrcOptions options = (OrcRecordUpdater.OrcOptions)
new OrcRecordUpdater.OrcOptions(conf)
@@ -1228,8 +1201,9 @@ public class TestOrcRawRecordMerger {
final int BUCKET_PROPERTY = BucketCodec.V1.encode(options);
options.orcOptions(OrcFile.writerOptions(conf)
- .stripeSize(1).blockPadding(false).compress(CompressionKind.NONE)
- .memory(mgr).batchSize(2));
+ .stripeSize(1).blockPadding(false)
+ .compress(CompressionKind.NONE)
+ .batchSize(2));
options.finalDestination(root);
RecordUpdater ru = of.getRecordUpdater(root, options);
String[] values= new String[]{"ignore.1", "0.1", "ignore.2", "ignore.3",