You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ab...@apache.org on 2019/11/30 21:59:17 UTC
[hive] branch master updated: HIVE-22555: Upgrade ORC version to
1.5.8 (Mustafa Iman via Prasanth Jayachandran)
This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new b09f9aa HIVE-22555: Upgrade ORC version to 1.5.8 (Mustafa Iman via Prasanth Jayachandran)
b09f9aa is described below
commit b09f9aaef4317f2d5ef4a2adcd30490dc99a5560
Author: Mustafa Iman <mu...@gmail.com>
AuthorDate: Sat Nov 30 22:51:35 2019 +0100
HIVE-22555: Upgrade ORC version to 1.5.8 (Mustafa Iman via Prasanth Jayachandran)
Signed-off-by: Laszlo Bodor <bo...@gmail.com>
---
pom.xml | 2 +-
.../apache/hadoop/hive/ql/io/orc/TestOrcFile.java | 81 ++++++++--------------
.../hive/ql/io/orc/TestOrcRawRecordMerger.java | 40 ++---------
3 files changed, 36 insertions(+), 87 deletions(-)
diff --git a/pom.xml b/pom.xml
index 7ac071d..ba87106 100644
--- a/pom.xml
+++ b/pom.xml
@@ -189,7 +189,7 @@
<log4j2.version>2.12.1</log4j2.version>
<mariadb.version>2.5.0</mariadb.version>
<opencsv.version>2.3</opencsv.version>
- <orc.version>1.5.6</orc.version>
+ <orc.version>1.5.8</orc.version>
<mockito-core.version>1.10.19</mockito-core.version>
<powermock.version>1.7.4</powermock.version>
<mina.version>2.0.0-M5</mina.version>
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
index 2931c04..220431a 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
@@ -1928,52 +1928,22 @@ public class TestOrcFile {
new MiddleStruct(inner, inner2), list(), map(inner,inner2));
}
- private static class MyMemoryManager implements MemoryManager {
- final long totalSpace;
- double rate;
- Path path = null;
- long lastAllocation = 0;
- int rows = 0;
- MemoryManager.Callback callback;
-
- MyMemoryManager(Configuration conf, long totalSpace, double rate) {
- this.totalSpace = totalSpace;
- this.rate = rate;
- }
-
- @Override
- public void addWriter(Path path, long requestedAllocation,
- MemoryManager.Callback callback) {
- this.path = path;
- this.lastAllocation = requestedAllocation;
- this.callback = callback;
- }
-
- @Override
- public synchronized void removeWriter(Path path) {
- this.path = null;
- this.lastAllocation = 0;
- }
-
- @Override
- public void addedRow(int count) throws IOException {
- rows += count;
- if (rows >= 100) {
- callback.checkMemory(rate);
- rows = 0;
- }
- }
- }
-
@Test
public void testMemoryManagementV11() throws Exception {
+ OrcConf.ROWS_BETWEEN_CHECKS.setLong(conf, 100);
+ final long poolSize = 50_000;
ObjectInspector inspector;
synchronized (TestOrcFile.class) {
inspector = ObjectInspectorFactory.getReflectionObjectInspector
(InnerStruct.class,
ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
}
- MyMemoryManager memory = new MyMemoryManager(conf, 10000, 0.1);
+ MemoryManager memoryManager = new MemoryManagerImpl(poolSize);
+ // set up 10 files that all request the full size.
+ MemoryManager.Callback ignore = newScale -> false;
+ for(int f=0; f < 9; ++f) {
+ memoryManager.addWriter(new Path("file-" + f), poolSize, ignore);
+ }
Writer writer = OrcFile.createWriter(testFilePath,
OrcFile.writerOptions(conf)
.inspector(inspector)
@@ -1981,15 +1951,14 @@ public class TestOrcFile {
.stripeSize(50000)
.bufferSize(100)
.rowIndexStride(0)
- .memory(memory)
+ .memory(memoryManager)
.batchSize(100)
.version(OrcFile.Version.V_0_11));
- assertEquals(testFilePath, memory.path);
+ assertEquals(0.1, ((MemoryManagerImpl) memoryManager).getAllocationScale());
for(int i=0; i < 2500; ++i) {
writer.addRow(new InnerStruct(i*300, Integer.toHexString(10*i)));
}
writer.close();
- assertEquals(null, memory.path);
Reader reader = OrcFile.createReader(testFilePath,
OrcFile.readerOptions(conf).filesystem(fs));
int i = 0;
@@ -2004,29 +1973,35 @@ public class TestOrcFile {
@Test
public void testMemoryManagementV12() throws Exception {
+ OrcConf.ROWS_BETWEEN_CHECKS.setLong(conf, 100);
+ final long poolSize = 50_000;
ObjectInspector inspector;
synchronized (TestOrcFile.class) {
inspector = ObjectInspectorFactory.getReflectionObjectInspector
(InnerStruct.class,
ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
}
- MyMemoryManager memory = new MyMemoryManager(conf, 10000, 0.1);
+ MemoryManager memoryManager = new MemoryManagerImpl(poolSize);
+ // set up 10 files that all request the full size.
+ MemoryManager.Callback ignore = newScale -> false;
+ for(int f=0; f < 9; ++f) {
+ memoryManager.addWriter(new Path("file-" + f), poolSize, ignore);
+ }
Writer writer = OrcFile.createWriter(testFilePath,
- OrcFile.writerOptions(conf)
- .inspector(inspector)
- .compress(CompressionKind.NONE)
- .stripeSize(50000)
- .bufferSize(100)
- .rowIndexStride(0)
- .memory(memory)
- .batchSize(100)
- .version(OrcFile.Version.V_0_12));
- assertEquals(testFilePath, memory.path);
+ OrcFile.writerOptions(conf)
+ .inspector(inspector)
+ .compress(CompressionKind.NONE)
+ .stripeSize(50000)
+ .bufferSize(100)
+ .rowIndexStride(0)
+ .memory(memoryManager)
+ .batchSize(100)
+ .version(OrcFile.Version.V_0_12));
+ assertEquals(0.1, ((MemoryManagerImpl) memoryManager).getAllocationScale());
for(int i=0; i < 2500; ++i) {
writer.addRow(new InnerStruct(i*300, Integer.toHexString(10*i)));
}
writer.close();
- assertEquals(null, memory.path);
Reader reader = OrcFile.createReader(testFilePath,
OrcFile.readerOptions(conf).filesystem(fs));
int i = 0;
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
index d4fd773..17ceede 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
@@ -25,9 +25,8 @@ import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.io.BucketCodec;
import org.apache.orc.CompressionKind;
-import org.apache.orc.MemoryManager;
+import org.apache.orc.OrcConf;
import org.apache.orc.StripeInformation;
-import org.apache.orc.impl.MemoryManagerImpl;
import org.apache.orc.impl.OrcAcidUtils;
import org.junit.Assert;
import org.junit.Rule;
@@ -1068,6 +1067,7 @@ public class TestOrcRawRecordMerger {
public void testRecordReaderOldBaseAndDelta() throws Exception {
final int BUCKET = 10;
Configuration conf = new Configuration();
+ OrcConf.ROWS_BETWEEN_CHECKS.setLong(conf, 2);
OrcOutputFormat of = new OrcOutputFormat();
FileSystem fs = FileSystem.getLocal(conf);
Path root = new Path(tmpDir, "testOldBaseAndDelta").makeQualified(fs);
@@ -1077,25 +1077,11 @@ public class TestOrcRawRecordMerger {
inspector = ObjectInspectorFactory.getReflectionObjectInspector
(BigRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
}
-
- // write the base
- MemoryManager mgr = new MemoryManagerImpl(conf){
- int rowsAddedSinceCheck = 0;
-
- @Override
- public synchronized void addedRow(int rows) throws IOException {
- rowsAddedSinceCheck += rows;
- if (rowsAddedSinceCheck >= 2) {
- notifyWriters();
- rowsAddedSinceCheck = 0;
- }
- }
- };
// make 5 stripes with 2 rows each
Writer writer = OrcFile.createWriter(new Path(root, "0000010_0"),
OrcFile.writerOptions(conf).inspector(inspector).fileSystem(fs)
.blockPadding(false).bufferSize(10000).compress(CompressionKind.NONE)
- .stripeSize(1).memory(mgr).batchSize(2).version(OrcFile.Version.V_0_11));
+ .stripeSize(1).batchSize(2).version(OrcFile.Version.V_0_11));
String[] values= new String[]{"ignore.1", "0.1", "ignore.2", "ignore.3",
"2.0", "2.1", "3.0", "ignore.4", "ignore.5", "ignore.6"};
for(int i=0; i < values.length; ++i) {
@@ -1201,6 +1187,7 @@ public class TestOrcRawRecordMerger {
public void testRecordReaderNewBaseAndDelta() throws Exception {
final int BUCKET = 11;
Configuration conf = new Configuration();
+ OrcConf.ROWS_BETWEEN_CHECKS.setLong(conf, 2);
OrcOutputFormat of = new OrcOutputFormat();
FileSystem fs = FileSystem.getLocal(conf);
Path root = new Path(tmpDir, "testRecordReaderNewBaseAndDelta").makeQualified(fs);
@@ -1211,20 +1198,6 @@ public class TestOrcRawRecordMerger {
(BigRow.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
}
- // write the base
- MemoryManager mgr = new MemoryManagerImpl(conf){
- int rowsAddedSinceCheck = 0;
-
- @Override
- public synchronized void addedRow(int rows) throws IOException {
- rowsAddedSinceCheck += rows;
- if (rowsAddedSinceCheck >= 2) {
- notifyWriters();
- rowsAddedSinceCheck = 0;
- }
- }
- };
-
// make 5 stripes with 2 rows each
OrcRecordUpdater.OrcOptions options = (OrcRecordUpdater.OrcOptions)
new OrcRecordUpdater.OrcOptions(conf)
@@ -1234,8 +1207,9 @@ public class TestOrcRawRecordMerger {
final int BUCKET_PROPERTY = BucketCodec.V1.encode(options);
options.orcOptions(OrcFile.writerOptions(conf)
- .stripeSize(1).blockPadding(false).compress(CompressionKind.NONE)
- .memory(mgr).batchSize(2));
+ .stripeSize(1).blockPadding(false)
+ .compress(CompressionKind.NONE)
+ .batchSize(2));
options.finalDestination(root);
RecordUpdater ru = of.getRecordUpdater(root, options);
String[] values= new String[]{"ignore.1", "0.1", "ignore.2", "ignore.3",