You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by bl...@apache.org on 2017/01/19 01:27:21 UTC
[10/50] [abbrv] parquet-mr git commit: PARQUET-431: Make
ParquetOutputFormat.memoryManager volatile
PARQUET-431: Make ParquetOutputFormat.memoryManager volatile
Currently ParquetOutputFormat.getRecordWriter() contains an unsynchronized lazy initialization of the non-volatile static field *memoryManager*.
Because the compiler or processor may reorder instructions, threads are not guaranteed to see a completely initialized object, when ParquetOutputFormat.getRecordWriter() is called by multiple threads.
This PR makes *memoryManager* volatile to correct the problem.
Author: Liwei Lin <pr...@gmail.com>
Author: proflin <pr...@gmail.com>
Closes #313 from proflin/PARQUET-431 and squashes the following commits:
1aa4a44 [Liwei Lin] empty commit to trigger CI
5e94fa3 [Liwei Lin] Remove the volatile modifier for memoryManager
d54bb99 [Liwei Lin] Undo the Deprecated anotation
fd1df4e [Liwei Lin] Adds synchronization around the creation of memoryManager as well as getMemoryManager()
615aa5a [proflin] PARQUET-431
Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/83c6cd5f
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/83c6cd5f
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/83c6cd5f
Branch: refs/heads/parquet-1.8.x
Commit: 83c6cd5f0bbbf13a058163cef582b929b1750c05
Parents: 60bbf8e
Author: Liwei Lin <pr...@gmail.com>
Authored: Mon Feb 15 16:37:04 2016 -0800
Committer: Ryan Blue <bl...@apache.org>
Committed: Mon Jan 9 16:54:53 2017 -0800
----------------------------------------------------------------------
.../org/apache/parquet/hadoop/ParquetOutputFormat.java | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/83c6cd5f/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java
index 419bd94..6cd4c90 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java
@@ -345,9 +345,12 @@ public class ParquetOutputFormat<T> extends FileOutputFormat<Void, T> {
MemoryManager.DEFAULT_MEMORY_POOL_RATIO);
long minAllocation = conf.getLong(ParquetOutputFormat.MIN_MEMORY_ALLOCATION,
MemoryManager.DEFAULT_MIN_MEMORY_ALLOCATION);
- if (memoryManager == null) {
- memoryManager = new MemoryManager(maxLoad, minAllocation);
- } else if (memoryManager.getMemoryPoolRatio() != maxLoad) {
+ synchronized (ParquetOutputFormat.class) {
+ if (memoryManager == null) {
+ memoryManager = new MemoryManager(maxLoad, minAllocation);
+ }
+ }
+ if (memoryManager.getMemoryPoolRatio() != maxLoad) {
LOG.warn("The configuration " + MEMORY_POOL_RATIO + " has been set. It should not " +
"be reset by the new value: " + maxLoad);
}
@@ -392,13 +395,12 @@ public class ParquetOutputFormat<T> extends FileOutputFormat<Void, T> {
return committer;
}
-
/**
* This memory manager is for all the real writers (InternalParquetRecordWriter) in one task.
*/
private static MemoryManager memoryManager;
- public static MemoryManager getMemoryManager() {
+ public synchronized static MemoryManager getMemoryManager() {
return memoryManager;
}
}