You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mh...@apache.org on 2020/11/04 22:39:56 UTC
[asterixdb] 01/03: [NO ISSUE][STO] Log bulkloader state on failure
This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit b1ae88e7e6bae81b6e9de5932ea16c8ffd077320
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Sat Oct 31 23:35:56 2020 +0300
[NO ISSUE][STO] Log bulkloader state on failure
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Log BTree bulkloader state when an exception is encountered.
Change-Id: I00a69d4883ae7a69c28a201408504049dd70ffb5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/8683
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../hyracks/hyracks-storage-am-btree/pom.xml | 13 +++++++++++
.../hyracks/storage/am/btree/impls/BTree.java | 26 +++++++++++++++++++++-
.../hyracks/hyracks-storage-am-common/pom.xml | 4 ++++
.../storage/am/common/api/ITreeIndexFrame.java | 13 +++++++++++
.../am/common/frames/TreeIndexNSMFrame.java | 20 +++++++++++++++++
5 files changed, 75 insertions(+), 1 deletion(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/pom.xml
index fa74184..45330e8 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/pom.xml
@@ -107,5 +107,18 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-util</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
index 3a062af..f62f2d1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
@@ -69,11 +69,16 @@ import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.buffercache.IPageWriteCallback;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.util.JSONUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
public class BTree extends AbstractTreeIndex {
public static final float DEFAULT_FILL_FACTOR = 0.7f;
-
+ private static final Logger LOGGER = LogManager.getLogger();
private static final long RESTART_OP = Long.MIN_VALUE;
private static final long FULL_RESTART_OP = Long.MIN_VALUE + 1;
private static final int MAX_RESTARTS = 10;
@@ -1086,6 +1091,7 @@ public class BTree extends AbstractTreeIndex {
}
((IBTreeLeafFrame) leafFrame).insertSorted(tuple);
} catch (HyracksDataException | RuntimeException e) {
+ logState(tuple, e);
handleException();
throw e;
}
@@ -1194,6 +1200,24 @@ public class BTree extends AbstractTreeIndex {
public void abort() throws HyracksDataException {
super.handleException();
}
+
+ private void logState(ITupleReference tuple, Exception e) {
+ try {
+ ObjectNode state = JSONUtil.createObject();
+ state.set("leafFrame", leafFrame.getState());
+ state.set("interiorFrame", interiorFrame.getState());
+ int tupleSize = Math.max(leafFrame.getBytesRequiredToWriteTuple(tuple),
+ interiorFrame.getBytesRequiredToWriteTuple(tuple));
+ state.put("tupleSize", tupleSize);
+ state.put("spaceNeeded", tupleWriter.bytesRequired(tuple) + slotSize);
+ state.put("spaceUsed", leafFrame.getBuffer().capacity() - leafFrame.getTotalFreeSpace());
+ state.put("leafMaxBytes", leafMaxBytes);
+ state.put("maxTupleSize", maxTupleSize);
+ LOGGER.error("failed to add tuple {}", state, e);
+ } catch (Throwable t) {
+ e.addSuppressed(t);
+ }
+ }
}
@SuppressWarnings("rawtypes")
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-common/pom.xml
index 31463b7..703deb8 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/pom.xml
@@ -109,5 +109,9 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java
index dc59612..18d5653 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java
@@ -28,6 +28,9 @@ import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper;
+import org.apache.hyracks.util.JSONUtil;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
public interface ITreeIndexFrame {
@@ -122,4 +125,14 @@ public interface ITreeIndexFrame {
public ITupleReference getLeftmostTuple() throws HyracksDataException;
public ITupleReference getRightmostTuple() throws HyracksDataException;
+
+ default ObjectNode getState() {
+ ObjectNode json = JSONUtil.createObject();
+ json.put("tupleCount", getTupleCount());
+ json.put("freeSpaceOff", getFreeSpaceOff());
+ json.put("level", getLevel());
+ json.put("pageLsn", getPageLsn());
+ json.put("totalFreeSpace", getTotalFreeSpace());
+ return json;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
index 6106358..08d4564 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
@@ -31,6 +31,10 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import org.apache.hyracks.storage.am.common.ophelpers.SlotOffTupleOff;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.util.JSONUtil;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.node.ObjectNode;
public abstract class TreeIndexNSMFrame implements ITreeIndexFrame {
@@ -354,4 +358,20 @@ public abstract class TreeIndexNSMFrame implements ITreeIndexFrame {
return frameTuple;
}
}
+
+ @Override
+ public ObjectNode getState() {
+ ObjectNode state = ITreeIndexFrame.super.getState();
+ state.put("largeFlag", getLargeFlag());
+ return state;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return JSONUtil.convertNode(getState());
+ } catch (JsonProcessingException e) {
+ return "failed to convert json";
+ }
+ }
}