You are viewing a plain text version of this content. The canonical link for it is here.

Posted to notifications@asterixdb.apache.org by "Michael Blow (Code Review)" <do...@asterixdb.incubator.apache.org> on 2019/04/05 12:45:16 UTC

Change in asterixdb[stabilization-f69489]: WIP: indicate null/misisng value fix in index checkpoint

Michael Blow has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/3324

Change subject: WIP: indicate null/misisng value fix in index checkpoint
......................................................................

WIP: indicate null/misisng value fix in index checkpoint

Change-Id: Id93ab9d16887b37cf6c0d011950e7c57f1a1d646
---
M asterixdb/asterix-common/pom.xml
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexCheckpoint.java
2 files changed, 21 insertions(+), 6 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/24/3324/1

diff --git a/asterixdb/asterix-common/pom.xml b/asterixdb/asterix-common/pom.xml
index 3945756..c9d7ac6 100644
--- a/asterixdb/asterix-common/pom.xml
+++ b/asterixdb/asterix-common/pom.xml
@@ -290,5 +290,9 @@
       <groupId>org.apache.hyracks</groupId>
       <artifactId>hyracks-control-nc</artifactId>
     </dependency>
+    <dependency>
+      <groupId>it.unimi.dsi</groupId>
+      <artifactId>fastutil</artifactId>
+    </dependency>
   </dependencies>
 </project>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexCheckpoint.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexCheckpoint.java
index cb34600..05056cc 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexCheckpoint.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexCheckpoint.java
@@ -19,27 +19,33 @@
 package org.apache.asterix.common.storage;
 
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.function.LongPredicate;
 
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
+import it.unimi.dsi.fastutil.longs.Long2LongMap;
+import it.unimi.dsi.fastutil.longs.Long2LongOpenHashMap;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
 public class IndexCheckpoint {
 
     private static final Logger LOGGER = LogManager.getLogger();
     private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
     private static final long INITIAL_CHECKPOINT_ID = 0;
+    // TODO(mblow): remove this marker & related logic once we no longer are able to read indexes prior to the fix
+    private static final long HAS_NULL_MISSING_VALUES_FIX = -1;
     private long id;
     private long validComponentSequence;
     private long lowWatermark;
     private long lastComponentId;
-    private Map<Long, Long> masterNodeFlushMap;
+    private Long2LongMap masterNodeFlushMap;
 
     public static IndexCheckpoint first(long lastComponentSequence, long lowWatermark, long validComponentId) {
         IndexCheckpoint firstCheckpoint = new IndexCheckpoint();
@@ -47,7 +53,8 @@
         firstCheckpoint.lowWatermark = lowWatermark;
         firstCheckpoint.validComponentSequence = lastComponentSequence;
         firstCheckpoint.lastComponentId = validComponentId;
-        firstCheckpoint.masterNodeFlushMap = new HashMap<>();
+        firstCheckpoint.masterNodeFlushMap = new Long2LongOpenHashMap();
+        firstCheckpoint.masterNodeFlushMap.put(HAS_NULL_MISSING_VALUES_FIX, HAS_NULL_MISSING_VALUES_FIX);
         return firstCheckpoint;
     }
 
@@ -66,7 +73,7 @@
         next.validComponentSequence = validComponentSequence;
         next.masterNodeFlushMap = latest.getMasterNodeFlushMap();
         // remove any lsn from the map that wont be used anymore
-        next.masterNodeFlushMap.values().removeIf(lsn -> lsn <= lowWatermark);
+        next.masterNodeFlushMap.values().removeIf((LongPredicate)(lsn -> lsn <= lowWatermark && lsn != HAS_NULL_MISSING_VALUES_FIX));
         return next;
     }
 
@@ -86,7 +93,7 @@
         return lastComponentId;
     }
 
-    public Map<Long, Long> getMasterNodeFlushMap() {
+    public Long2LongMap getMasterNodeFlushMap() {
         return masterNodeFlushMap;
     }
 
@@ -94,6 +101,10 @@
         return id;
     }
 
+    public boolean hasNullMissingValuesFix() {
+        return masterNodeFlushMap.containsKey(HAS_NULL_MISSING_VALUES_FIX);
+    }
+
     public String asJson() throws HyracksDataException {
         try {
             return OBJECT_MAPPER.writeValueAsString(this);

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/3324
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Id93ab9d16887b37cf6c0d011950e7c57f1a1d646
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: stabilization-f69489
Gerrit-Owner: Michael Blow <mb...@apache.org>