You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2021/05/26 23:43:31 UTC
[asterixdb] 25/38: [NO ISSUE][*DB] Enable large parsing resources
to be freed on memory pressure
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 5d02b03854911d7fb889b078f0ada37b282261f9
Author: Michael Blow <mb...@apache.org>
AuthorDate: Thu May 6 16:36:30 2021 -0400
[NO ISSUE][*DB] Enable large parsing resources to be freed on memory pressure
Change-Id: Ieaca566f765a5d3531baff8128d6ff03c95956e2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11365
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Till Westmann <ti...@apache.org>
---
.../{ObjectPool.java => AbstractObjectPool.java} | 28 +++++++---------
.../external/parser/jackson/IObjectPool.java | 25 +++++---------
.../external/parser/jackson/ObjectPool.java | 34 +++++--------------
.../external/parser/jackson/ParserContext.java | 8 ++---
.../{ObjectPool.java => SoftObjectPool.java} | 38 +++++++---------------
.../apache/hyracks/util/string/UTF8StringUtil.java | 11 ++++---
.../hyracks/util/string/UTF8StringWriter.java | 3 +-
7 files changed, 52 insertions(+), 95 deletions(-)
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/AbstractObjectPool.java
similarity index 75%
copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
copy to asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/AbstractObjectPool.java
index 8945e71..ef75688 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/AbstractObjectPool.java
@@ -27,39 +27,35 @@ import org.apache.asterix.om.util.container.IObjectFactory;
* Object pool for DFS traversal mode, which allows to recycle objects
* as soon as it is not needed.
*/
-public class ObjectPool<E, T> {
+public abstract class AbstractObjectPool<E, T, Q> implements IObjectPool<E> {
private final IObjectFactory<E, T> objectFactory;
- private final Queue<E> recycledObjects;
- private final T element;
+ private final Queue<Q> recycledObjects;
+ private final T param;
- public ObjectPool() {
- this(null, null);
- }
-
- public ObjectPool(IObjectFactory<E, T> objectFactory) {
- this(objectFactory, null);
- }
-
- public ObjectPool(IObjectFactory<E, T> objectFactory, T element) {
+ protected AbstractObjectPool(IObjectFactory<E, T> objectFactory, T param) {
this.objectFactory = objectFactory;
recycledObjects = new ArrayDeque<>();
- this.element = element;
+ this.param = param;
}
public E getInstance() {
- E instance = recycledObjects.poll();
+ E instance = unwrap(recycledObjects.poll());
if (objectFactory != null && instance == null) {
- instance = objectFactory.create(element);
+ instance = objectFactory.create(param);
}
return instance;
}
public void recycle(E object) {
if (object != null) {
- recycledObjects.add(object);
+ recycledObjects.add(wrap(object));
}
}
+ protected abstract E unwrap(Q element);
+
+ protected abstract Q wrap(E element);
+
@Override
public String toString() {
return recycledObjects.toString();
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/IObjectPool.java
similarity index 57%
copy from hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java
copy to asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/IObjectPool.java
index 563f865..a2990ec 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/IObjectPool.java
@@ -16,23 +16,14 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.util.string;
+package org.apache.asterix.external.parser.jackson;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.Serializable;
-
-public class UTF8StringWriter implements Serializable {
-
- private static final long serialVersionUID = 1L;
- transient byte[] tempBytes;
-
- public final void writeUTF8(CharSequence str, DataOutput out) throws IOException {
- UTF8StringUtil.writeUTF8(str, out, this);
- }
-
- public final void writeUTF8(char[] buffer, int start, int length, DataOutput out) throws IOException {
- UTF8StringUtil.writeUTF8(buffer, start, length, out, this);
- }
+/**
+ * Object pool for DFS traversal mode, which allows to recycle objects
+ * as soon as it is not needed.
+ */
+public interface IObjectPool<E> {
+ E getInstance();
+ void recycle(E object);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
index 8945e71..864d9a9 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
@@ -18,20 +18,13 @@
*/
package org.apache.asterix.external.parser.jackson;
-import java.util.ArrayDeque;
-import java.util.Queue;
-
import org.apache.asterix.om.util.container.IObjectFactory;
/**
* Object pool for DFS traversal mode, which allows to recycle objects
* as soon as it is not needed.
*/
-public class ObjectPool<E, T> {
- private final IObjectFactory<E, T> objectFactory;
- private final Queue<E> recycledObjects;
- private final T element;
-
+public class ObjectPool<E, T> extends AbstractObjectPool<E, T, E> {
public ObjectPool() {
this(null, null);
}
@@ -40,28 +33,17 @@ public class ObjectPool<E, T> {
this(objectFactory, null);
}
- public ObjectPool(IObjectFactory<E, T> objectFactory, T element) {
- this.objectFactory = objectFactory;
- recycledObjects = new ArrayDeque<>();
- this.element = element;
+ public ObjectPool(IObjectFactory<E, T> objectFactory, T param) {
+ super(objectFactory, param);
}
- public E getInstance() {
- E instance = recycledObjects.poll();
- if (objectFactory != null && instance == null) {
- instance = objectFactory.create(element);
- }
- return instance;
- }
-
- public void recycle(E object) {
- if (object != null) {
- recycledObjects.add(object);
- }
+ @Override
+ protected E unwrap(E wrapped) {
+ return wrapped;
}
@Override
- public String toString() {
- return recycledObjects.toString();
+ protected E wrap(E element) {
+ return element;
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ParserContext.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ParserContext.java
index d0b79b1..ef9ff08 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ParserContext.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ParserContext.java
@@ -51,7 +51,7 @@ import org.apache.hyracks.util.string.UTF8StringWriter;
public class ParserContext {
private static final int SERIALIZED_FIELDNAME_MAP_MAX_SIZE = 128;
- private final ObjectPool<IARecordBuilder, ATypeTag> objectBuilderPool;
+ private final IObjectPool<IARecordBuilder> objectBuilderPool;
private final ObjectPool<IAsterixListBuilder, ATypeTag> arrayBuilderPool;
/**
@@ -61,7 +61,7 @@ public class ParserContext {
* <p>
* Scalar value 5 is written 4 times in tempBuffer("d") then tempBuffer("c") ... tempBuffer("a")
*/
- private final ObjectPool<IMutableValueStorage, ATypeTag> tempBufferPool;
+ private final IObjectPool<IMutableValueStorage> tempBufferPool;
private final ObjectPool<BitSet, Void> nullBitmapPool;
private final Map<String, IMutableValueStorage> serializedFieldNames;
private final ISerializerDeserializer<AString> stringSerDe;
@@ -76,9 +76,9 @@ public class ParserContext {
@SuppressWarnings("unchecked")
public ParserContext(boolean allocateModfiedUTF8Writer) {
- objectBuilderPool = new ObjectPool<>(new RecordBuilderFactory());
+ objectBuilderPool = new SoftObjectPool<>(new RecordBuilderFactory());
arrayBuilderPool = new ObjectPool<>(new ListBuilderFactory(), ATypeTag.ARRAY);
- tempBufferPool = new ObjectPool<>(new AbvsBuilderFactory());
+ tempBufferPool = new SoftObjectPool<>(new AbvsBuilderFactory());
nullBitmapPool = new ObjectPool<>();
serializedFieldNames = new LRUMap<>(SERIALIZED_FIELDNAME_MAP_MAX_SIZE);
stringSerDe = SerializerDeserializerProvider.INSTANCE.getAStringSerializerDeserializer();
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/SoftObjectPool.java
similarity index 55%
copy from asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
copy to asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/SoftObjectPool.java
index 8945e71..91655db 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/ObjectPool.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/jackson/SoftObjectPool.java
@@ -18,8 +18,7 @@
*/
package org.apache.asterix.external.parser.jackson;
-import java.util.ArrayDeque;
-import java.util.Queue;
+import java.lang.ref.SoftReference;
import org.apache.asterix.om.util.container.IObjectFactory;
@@ -27,41 +26,26 @@ import org.apache.asterix.om.util.container.IObjectFactory;
* Object pool for DFS traversal mode, which allows to recycle objects
* as soon as it is not needed.
*/
-public class ObjectPool<E, T> {
- private final IObjectFactory<E, T> objectFactory;
- private final Queue<E> recycledObjects;
- private final T element;
-
- public ObjectPool() {
+public class SoftObjectPool<E, T> extends AbstractObjectPool<E, T, SoftReference<E>> {
+ public SoftObjectPool() {
this(null, null);
}
- public ObjectPool(IObjectFactory<E, T> objectFactory) {
+ public SoftObjectPool(IObjectFactory<E, T> objectFactory) {
this(objectFactory, null);
}
- public ObjectPool(IObjectFactory<E, T> objectFactory, T element) {
- this.objectFactory = objectFactory;
- recycledObjects = new ArrayDeque<>();
- this.element = element;
- }
-
- public E getInstance() {
- E instance = recycledObjects.poll();
- if (objectFactory != null && instance == null) {
- instance = objectFactory.create(element);
- }
- return instance;
+ public SoftObjectPool(IObjectFactory<E, T> objectFactory, T element) {
+ super(objectFactory, element);
}
- public void recycle(E object) {
- if (object != null) {
- recycledObjects.add(object);
- }
+ @Override
+ protected E unwrap(SoftReference<E> wrapped) {
+ return wrapped == null ? null : wrapped.get();
}
@Override
- public String toString() {
- return recycledObjects.toString();
+ protected SoftReference<E> wrap(E element) {
+ return new SoftReference<>(element);
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
index f96ed72..38de4ac 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringUtil.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -25,6 +25,7 @@ import java.io.EOFException;
import java.io.IOException;
import java.io.OutputStream;
import java.io.UTFDataFormatException;
+import java.lang.ref.SoftReference;
import org.apache.hyracks.util.encoding.VarLenIntEncoderDecoder;
@@ -690,10 +691,12 @@ public class UTF8StringUtil {
if (writer == null) {
tempBytes = new byte[utflen + 5];
} else {
- if (writer.tempBytes == null || writer.tempBytes.length < utflen + 5) {
- writer.tempBytes = new byte[utflen + 5];
+ byte[] writerTempBytes = writer.tempBytesRef != null ? writer.tempBytesRef.get() : null;
+ if (writerTempBytes == null || writerTempBytes.length < utflen + 5) {
+ writerTempBytes = new byte[utflen + 5];
+ writer.tempBytesRef = new SoftReference<>(writerTempBytes);
}
- tempBytes = writer.tempBytes;
+ tempBytes = writerTempBytes;
}
return tempBytes;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java
index 563f865..a0cc7d0 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/string/UTF8StringWriter.java
@@ -21,11 +21,12 @@ package org.apache.hyracks.util.string;
import java.io.DataOutput;
import java.io.IOException;
import java.io.Serializable;
+import java.lang.ref.SoftReference;
public class UTF8StringWriter implements Serializable {
private static final long serialVersionUID = 1L;
- transient byte[] tempBytes;
+ transient SoftReference<byte[]> tempBytesRef;
public final void writeUTF8(CharSequence str, DataOutput out) throws IOException {
UTF8StringUtil.writeUTF8(str, out, this);