You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2017/02/06 21:10:13 UTC
orc git commit: ORC-141. Make setting a memory manager for the writer
public. (omalley)
Repository: orc
Updated Branches:
refs/heads/master 82adb43c5 -> 41f2d7335
ORC-141. Make setting a memory manager for the writer public. (omalley)
Fixes #89
Signed-off-by: Owen O'Malley <om...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/41f2d733
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/41f2d733
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/41f2d733
Branch: refs/heads/master
Commit: 41f2d733593cc50fd0c5ddc79c4eac745fe4a535
Parents: 82adb43
Author: Owen O'Malley <om...@apache.org>
Authored: Sat Feb 4 10:04:00 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Mon Feb 6 13:09:35 2017 -0800
----------------------------------------------------------------------
.../src/java/org/apache/orc/MemoryManager.java | 68 ++++++
java/core/src/java/org/apache/orc/OrcFile.java | 7 +-
.../java/org/apache/orc/impl/MemoryManager.java | 212 -------------------
.../org/apache/orc/impl/MemoryManagerImpl.java | 204 ++++++++++++++++++
.../java/org/apache/orc/impl/WriterImpl.java | 1 +
.../test/org/apache/orc/TestVectorOrcFile.java | 16 +-
.../org/apache/orc/impl/TestMemoryManager.java | 10 +-
7 files changed, 283 insertions(+), 235 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/orc/blob/41f2d733/java/core/src/java/org/apache/orc/MemoryManager.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/MemoryManager.java b/java/core/src/java/org/apache/orc/MemoryManager.java
new file mode 100644
index 0000000..3afd3f5
--- /dev/null
+++ b/java/core/src/java/org/apache/orc/MemoryManager.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc;
+
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+
+/**
+ * A memory manager that keeps a global context of how many ORC
+ * writers there are and manages the memory between them. For use cases with
+ * dynamic partitions, it is easy to end up with many writers in the same task.
+ * By managing the size of each allocation, we try to cut down the size of each
+ * allocation and keep the task from running out of memory.
+ *
+ * This class is not thread safe, but is re-entrant - ensure creation and all
+ * invocations are triggered from the same thread.
+ */
+public interface MemoryManager {
+
+ interface Callback {
+ /**
+ * The writer needs to check its memory usage
+ * @param newScale the current scale factor for memory allocations
+ * @return true if the writer was over the limit
+ * @throws IOException
+ */
+ boolean checkMemory(double newScale) throws IOException;
+ }
+
+ /**
+ * Add a new writer's memory allocation to the pool. We use the path
+ * as a unique key to ensure that we don't get duplicates.
+ * @param path the file that is being written
+ * @param requestedAllocation the requested buffer size
+ */
+ void addWriter(Path path, long requestedAllocation,
+ Callback callback) throws IOException;
+
+ /**
+ * Remove the given writer from the pool.
+ * @param path the file that has been closed
+ */
+ void removeWriter(Path path) throws IOException;
+
+ /**
+ * Give the memory manager an opportunity for doing a memory check.
+ * @param rows number of rows added
+ * @throws IOException
+ */
+ void addedRow(int rows) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/orc/blob/41f2d733/java/core/src/java/org/apache/orc/OrcFile.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/OrcFile.java b/java/core/src/java/org/apache/orc/OrcFile.java
index cfabba9..e3d108a 100644
--- a/java/core/src/java/org/apache/orc/OrcFile.java
+++ b/java/core/src/java/org/apache/orc/OrcFile.java
@@ -30,7 +30,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.orc.impl.MemoryManager;
+import org.apache.orc.MemoryManager;
+import org.apache.orc.impl.MemoryManagerImpl;
import org.apache.orc.impl.OrcTail;
import org.apache.orc.impl.ReaderImpl;
import org.apache.orc.impl.WriterImpl;
@@ -514,7 +515,7 @@ public class OrcFile {
/**
* A package local option to set the memory manager.
*/
- protected WriterOptions memory(MemoryManager value) {
+ public WriterOptions memory(MemoryManager value) {
memoryManagerValue = value;
return this;
}
@@ -647,7 +648,7 @@ public class OrcFile {
memoryManager = new ThreadLocal<MemoryManager>() {
@Override
protected MemoryManager initialValue() {
- return new MemoryManager(conf);
+ return new MemoryManagerImpl(conf);
}
};
}
http://git-wip-us.apache.org/repos/asf/orc/blob/41f2d733/java/core/src/java/org/apache/orc/impl/MemoryManager.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/MemoryManager.java b/java/core/src/java/org/apache/orc/impl/MemoryManager.java
deleted file mode 100644
index 23f7533..0000000
--- a/java/core/src/java/org/apache/orc/impl/MemoryManager.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.orc.impl;
-
-import org.apache.orc.OrcConf;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.locks.ReentrantLock;
-
-/**
- * Implements a memory manager that keeps a global context of how many ORC
- * writers there are and manages the memory between them. For use cases with
- * dynamic partitions, it is easy to end up with many writers in the same task.
- * By managing the size of each allocation, we try to cut down the size of each
- * allocation and keep the task from running out of memory.
- *
- * This class is not thread safe, but is re-entrant - ensure creation and all
- * invocations are triggered from the same thread.
- */
-public class MemoryManager {
-
- private static final Logger LOG = LoggerFactory.getLogger(MemoryManager.class);
-
- /**
- * How often should we check the memory sizes? Measured in rows added
- * to all of the writers.
- */
- private static final int ROWS_BETWEEN_CHECKS = 5000;
- private final long totalMemoryPool;
- private final Map<Path, WriterInfo> writerList =
- new HashMap<Path, WriterInfo>();
- private long totalAllocation = 0;
- private double currentScale = 1;
- private int rowsAddedSinceCheck = 0;
- private final OwnedLock ownerLock = new OwnedLock();
-
- @SuppressWarnings("serial")
- private static class OwnedLock extends ReentrantLock {
- public Thread getOwner() {
- return super.getOwner();
- }
- }
-
- private static class WriterInfo {
- long allocation;
- Callback callback;
- WriterInfo(long allocation, Callback callback) {
- this.allocation = allocation;
- this.callback = callback;
- }
- }
-
- public interface Callback {
- /**
- * The writer needs to check its memory usage
- * @param newScale the current scale factor for memory allocations
- * @return true if the writer was over the limit
- * @throws IOException
- */
- boolean checkMemory(double newScale) throws IOException;
- }
-
- /**
- * Create the memory manager.
- * @param conf use the configuration to find the maximum size of the memory
- * pool.
- */
- public MemoryManager(Configuration conf) {
- double maxLoad = OrcConf.MEMORY_POOL.getDouble(conf);
- totalMemoryPool = Math.round(ManagementFactory.getMemoryMXBean().
- getHeapMemoryUsage().getMax() * maxLoad);
- ownerLock.lock();
- }
-
- /**
- * Light weight thread-safety check for multi-threaded access patterns
- */
- private void checkOwner() {
- if (!ownerLock.isHeldByCurrentThread()) {
- LOG.warn("Owner thread expected {}, got {}",
- ownerLock.getOwner(), Thread.currentThread());
- }
- }
-
- /**
- * Add a new writer's memory allocation to the pool. We use the path
- * as a unique key to ensure that we don't get duplicates.
- * @param path the file that is being written
- * @param requestedAllocation the requested buffer size
- */
- public void addWriter(Path path, long requestedAllocation,
- Callback callback) throws IOException {
- checkOwner();
- WriterInfo oldVal = writerList.get(path);
- // this should always be null, but we handle the case where the memory
- // manager wasn't told that a writer wasn't still in use and the task
- // starts writing to the same path.
- if (oldVal == null) {
- oldVal = new WriterInfo(requestedAllocation, callback);
- writerList.put(path, oldVal);
- totalAllocation += requestedAllocation;
- } else {
- // handle a new writer that is writing to the same path
- totalAllocation += requestedAllocation - oldVal.allocation;
- oldVal.allocation = requestedAllocation;
- oldVal.callback = callback;
- }
- updateScale(true);
- }
-
- /**
- * Remove the given writer from the pool.
- * @param path the file that has been closed
- */
- public void removeWriter(Path path) throws IOException {
- checkOwner();
- WriterInfo val = writerList.get(path);
- if (val != null) {
- writerList.remove(path);
- totalAllocation -= val.allocation;
- if (writerList.isEmpty()) {
- rowsAddedSinceCheck = 0;
- }
- updateScale(false);
- }
- if(writerList.isEmpty()) {
- rowsAddedSinceCheck = 0;
- }
- }
-
- /**
- * Get the total pool size that is available for ORC writers.
- * @return the number of bytes in the pool
- */
- public long getTotalMemoryPool() {
- return totalMemoryPool;
- }
-
- /**
- * The scaling factor for each allocation to ensure that the pool isn't
- * oversubscribed.
- * @return a fraction between 0.0 and 1.0 of the requested size that is
- * available for each writer.
- */
- public double getAllocationScale() {
- return currentScale;
- }
-
- /**
- * Give the memory manager an opportunity for doing a memory check.
- * @param rows number of rows added
- * @throws IOException
- */
- public void addedRow(int rows) throws IOException {
- rowsAddedSinceCheck += rows;
- if (rowsAddedSinceCheck >= ROWS_BETWEEN_CHECKS) {
- notifyWriters();
- }
- }
-
- /**
- * Notify all of the writers that they should check their memory usage.
- * @throws IOException
- */
- public void notifyWriters() throws IOException {
- checkOwner();
- LOG.debug("Notifying writers after " + rowsAddedSinceCheck);
- for(WriterInfo writer: writerList.values()) {
- boolean flushed = writer.callback.checkMemory(currentScale);
- if (LOG.isDebugEnabled() && flushed) {
- LOG.debug("flushed " + writer.toString());
- }
- }
- rowsAddedSinceCheck = 0;
- }
-
- /**
- * Update the currentScale based on the current allocation and pool size.
- * This also updates the notificationTrigger.
- * @param isAllocate is this an allocation?
- */
- private void updateScale(boolean isAllocate) throws IOException {
- if (totalAllocation <= totalMemoryPool) {
- currentScale = 1;
- } else {
- currentScale = (double) totalMemoryPool / totalAllocation;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/orc/blob/41f2d733/java/core/src/java/org/apache/orc/impl/MemoryManagerImpl.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/MemoryManagerImpl.java b/java/core/src/java/org/apache/orc/impl/MemoryManagerImpl.java
new file mode 100644
index 0000000..bd6c152
--- /dev/null
+++ b/java/core/src/java/org/apache/orc/impl/MemoryManagerImpl.java
@@ -0,0 +1,204 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.orc.impl;
+
+import org.apache.orc.MemoryManager;
+import org.apache.orc.OrcConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * Implements a memory manager that keeps a global context of how many ORC
+ * writers there are and manages the memory between them. For use cases with
+ * dynamic partitions, it is easy to end up with many writers in the same task.
+ * By managing the size of each allocation, we try to cut down the size of each
+ * allocation and keep the task from running out of memory.
+ *
+ * This class is not thread safe, but is re-entrant - ensure creation and all
+ * invocations are triggered from the same thread.
+ */
+public class MemoryManagerImpl implements MemoryManager {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MemoryManagerImpl.class);
+
+ /**
+ * How often should we check the memory sizes? Measured in rows added
+ * to all of the writers.
+ */
+ private static final int ROWS_BETWEEN_CHECKS = 5000;
+ private final long totalMemoryPool;
+ private final Map<Path, WriterInfo> writerList =
+ new HashMap<Path, WriterInfo>();
+ private long totalAllocation = 0;
+ private double currentScale = 1;
+ private int rowsAddedSinceCheck = 0;
+ private final OwnedLock ownerLock = new OwnedLock();
+
+ @SuppressWarnings("serial")
+ private static class OwnedLock extends ReentrantLock {
+ public Thread getOwner() {
+ return super.getOwner();
+ }
+ }
+
+ private static class WriterInfo {
+ long allocation;
+ Callback callback;
+ WriterInfo(long allocation, Callback callback) {
+ this.allocation = allocation;
+ this.callback = callback;
+ }
+ }
+
+ /**
+ * Create the memory manager.
+ * @param conf use the configuration to find the maximum size of the memory
+ * pool.
+ */
+ public MemoryManagerImpl(Configuration conf) {
+ double maxLoad = OrcConf.MEMORY_POOL.getDouble(conf);
+ totalMemoryPool = Math.round(ManagementFactory.getMemoryMXBean().
+ getHeapMemoryUsage().getMax() * maxLoad);
+ ownerLock.lock();
+ }
+
+ /**
+ * Light weight thread-safety check for multi-threaded access patterns
+ */
+ private void checkOwner() {
+ if (!ownerLock.isHeldByCurrentThread()) {
+ LOG.warn("Owner thread expected {}, got {}",
+ ownerLock.getOwner(), Thread.currentThread());
+ }
+ }
+
+ /**
+ * Add a new writer's memory allocation to the pool. We use the path
+ * as a unique key to ensure that we don't get duplicates.
+ * @param path the file that is being written
+ * @param requestedAllocation the requested buffer size
+ */
+ public void addWriter(Path path, long requestedAllocation,
+ Callback callback) throws IOException {
+ checkOwner();
+ WriterInfo oldVal = writerList.get(path);
+ // this should always be null, but we handle the case where the memory
+ // manager wasn't told that a writer wasn't still in use and the task
+ // starts writing to the same path.
+ if (oldVal == null) {
+ oldVal = new WriterInfo(requestedAllocation, callback);
+ writerList.put(path, oldVal);
+ totalAllocation += requestedAllocation;
+ } else {
+ // handle a new writer that is writing to the same path
+ totalAllocation += requestedAllocation - oldVal.allocation;
+ oldVal.allocation = requestedAllocation;
+ oldVal.callback = callback;
+ }
+ updateScale(true);
+ }
+
+ /**
+ * Remove the given writer from the pool.
+ * @param path the file that has been closed
+ */
+ public void removeWriter(Path path) throws IOException {
+ checkOwner();
+ WriterInfo val = writerList.get(path);
+ if (val != null) {
+ writerList.remove(path);
+ totalAllocation -= val.allocation;
+ if (writerList.isEmpty()) {
+ rowsAddedSinceCheck = 0;
+ }
+ updateScale(false);
+ }
+ if(writerList.isEmpty()) {
+ rowsAddedSinceCheck = 0;
+ }
+ }
+
+ /**
+ * Get the total pool size that is available for ORC writers.
+ * @return the number of bytes in the pool
+ */
+ public long getTotalMemoryPool() {
+ return totalMemoryPool;
+ }
+
+ /**
+ * The scaling factor for each allocation to ensure that the pool isn't
+ * oversubscribed.
+ * @return a fraction between 0.0 and 1.0 of the requested size that is
+ * available for each writer.
+ */
+ public double getAllocationScale() {
+ return currentScale;
+ }
+
+ /**
+ * Give the memory manager an opportunity for doing a memory check.
+ * @param rows number of rows added
+ * @throws IOException
+ */
+ @Override
+ public void addedRow(int rows) throws IOException {
+ rowsAddedSinceCheck += rows;
+ if (rowsAddedSinceCheck >= ROWS_BETWEEN_CHECKS) {
+ notifyWriters();
+ }
+ }
+
+ /**
+ * Notify all of the writers that they should check their memory usage.
+ * @throws IOException
+ */
+ public void notifyWriters() throws IOException {
+ checkOwner();
+ LOG.debug("Notifying writers after " + rowsAddedSinceCheck);
+ for(WriterInfo writer: writerList.values()) {
+ boolean flushed = writer.callback.checkMemory(currentScale);
+ if (LOG.isDebugEnabled() && flushed) {
+ LOG.debug("flushed " + writer.toString());
+ }
+ }
+ rowsAddedSinceCheck = 0;
+ }
+
+ /**
+ * Update the currentScale based on the current allocation and pool size.
+ * This also updates the notificationTrigger.
+ * @param isAllocate is this an allocation?
+ */
+ private void updateScale(boolean isAllocate) throws IOException {
+ if (totalAllocation <= totalMemoryPool) {
+ currentScale = 1;
+ } else {
+ currentScale = (double) totalMemoryPool / totalAllocation;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/orc/blob/41f2d733/java/core/src/java/org/apache/orc/impl/WriterImpl.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/WriterImpl.java b/java/core/src/java/org/apache/orc/impl/WriterImpl.java
index 7766408..925164d 100644
--- a/java/core/src/java/org/apache/orc/impl/WriterImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/WriterImpl.java
@@ -40,6 +40,7 @@ import org.apache.orc.BinaryColumnStatistics;
import org.apache.orc.ColumnStatistics;
import org.apache.orc.CompressionCodec;
import org.apache.orc.CompressionKind;
+import org.apache.orc.MemoryManager;
import org.apache.orc.OrcConf;
import org.apache.orc.OrcFile;
import org.apache.orc.OrcProto;
http://git-wip-us.apache.org/repos/asf/orc/blob/41f2d733/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
----------------------------------------------------------------------
diff --git a/java/core/src/test/org/apache/orc/TestVectorOrcFile.java b/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
index c3df813..b7fa8ee 100644
--- a/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
+++ b/java/core/src/test/org/apache/orc/TestVectorOrcFile.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
import org.apache.orc.impl.DataReaderProperties;
-import org.apache.orc.impl.MemoryManager;
import org.apache.orc.impl.OrcIndex;
import org.apache.orc.impl.RecordReaderImpl;
import org.apache.orc.impl.RecordReaderUtils;
@@ -59,7 +58,6 @@ import java.io.IOException;
import java.math.BigInteger;
import java.net.URL;
import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.sql.Date;
import java.sql.Timestamp;
@@ -2038,8 +2036,7 @@ public class TestVectorOrcFile {
new MiddleStruct(inner, inner2), list(), map(inner, inner2));
}
- private static class MyMemoryManager extends MemoryManager {
- final long totalSpace;
+ private static class MyMemoryManager implements MemoryManager {
double rate;
Path path = null;
long lastAllocation = 0;
@@ -2047,8 +2044,6 @@ public class TestVectorOrcFile {
Callback callback;
MyMemoryManager(Configuration conf, long totalSpace, double rate) {
- super(conf);
- this.totalSpace = totalSpace;
this.rate = rate;
}
@@ -2066,15 +2061,6 @@ public class TestVectorOrcFile {
this.lastAllocation = 0;
}
- @Override
- public long getTotalMemoryPool() {
- return totalSpace;
- }
-
- @Override
- public double getAllocationScale() {
- return rate;
- }
@Override
public void addedRow(int count) throws IOException {
http://git-wip-us.apache.org/repos/asf/orc/blob/41f2d733/java/core/src/test/org/apache/orc/impl/TestMemoryManager.java
----------------------------------------------------------------------
diff --git a/java/core/src/test/org/apache/orc/impl/TestMemoryManager.java b/java/core/src/test/org/apache/orc/impl/TestMemoryManager.java
index f48c545..b84e6ed 100644
--- a/java/core/src/test/org/apache/orc/impl/TestMemoryManager.java
+++ b/java/core/src/test/org/apache/orc/impl/TestMemoryManager.java
@@ -19,7 +19,7 @@ package org.apache.orc.impl;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.apache.orc.impl.MemoryManager;
+import org.apache.orc.MemoryManager;
import org.hamcrest.BaseMatcher;
import org.hamcrest.Description;
import org.junit.Test;
@@ -39,7 +39,7 @@ import static org.mockito.Mockito.verify;
public class TestMemoryManager {
private static final double ERROR = 0.000001;
- private static class NullCallback implements MemoryManager.Callback {
+ private static class NullCallback implements MemoryManagerImpl.Callback {
public boolean checkMemory(double newScale) {
return false;
}
@@ -48,7 +48,7 @@ public class TestMemoryManager {
@Test
public void testBasics() throws Exception {
Configuration conf = new Configuration();
- MemoryManager mgr = new MemoryManager(conf);
+ MemoryManagerImpl mgr = new MemoryManagerImpl(conf);
NullCallback callback = new NullCallback();
long poolSize = mgr.getTotalMemoryPool();
assertEquals(Math.round(ManagementFactory.getMemoryMXBean().
@@ -77,7 +77,7 @@ public class TestMemoryManager {
public void testConfig() throws Exception {
Configuration conf = new Configuration();
conf.set("hive.exec.orc.memory.pool", "0.9");
- MemoryManager mgr = new MemoryManager(conf);
+ MemoryManagerImpl mgr = new MemoryManagerImpl(conf);
long mem =
ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
System.err.print("Memory = " + mem);
@@ -114,7 +114,7 @@ public class TestMemoryManager {
@Test
public void testCallback() throws Exception {
Configuration conf = new Configuration();
- MemoryManager mgr = new MemoryManager(conf);
+ MemoryManagerImpl mgr = new MemoryManagerImpl(conf);
long pool = mgr.getTotalMemoryPool();
MemoryManager.Callback[] calls = new MemoryManager.Callback[20];
for(int i=0; i < calls.length; ++i) {