[GitHub] [ozone] hemantk-12 commented on a diff in pull request #3824: HDDS-7281. [Snapshot] Handle RocksDB compaction DAG persistence and reconstruction

["hemantk-12 (via GitHub)" <gi...@apache.org>]

hemantk-12 commented on code in PR #3824:
URL: https://github.com/apache/ozone/pull/3824#discussion_r1011056164


##########
hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java:
##########
@@ -688,130 +868,139 @@ public synchronized void printMutableGraph(
       iter.remove();
       iter = topLevelNodes.iterator();
     }
-    LOG.warn("src snap:" + srcSnapId);
-    LOG.warn("dest snap:" + destSnapId);
+    LOG.debug("src snap: " + srcSnapId);
+    LOG.debug("dest snap: " + destSnapId);
     for (CompactionNode n : allNodes) {
-      LOG.warn("Files are :" + n.fileName);
+      LOG.debug("Files are: " + n.fileName);
     }
   }
 
+  public MutableGraph<CompactionNode> getCompactionFwdDAG() {
+    return compactionDAGFwd;
+  }
 
-  public void createSnapshot(RocksDB rocksDB) throws InterruptedException {
-
-    LOG.warn("Current time is::" + System.currentTimeMillis());
-    long t1 = System.currentTimeMillis();
-
-    cpPath = cpPath + lastSnapshotCounter;
-    createCheckPoint(rocksDbPath, cpPath, rocksDB);
-    allSnapshots[lastSnapshotCounter] = new Snapshot(cpPath,
-    lastSnapshotPrefix, lastSnapshotCounter);
-
-    long t2 = System.currentTimeMillis();
-    LOG.warn("Current time is::" + t2);
-
-    LOG.warn("millisecond difference is ::" + (t2 - t1));
-   Thread.sleep(100);
-   ++lastSnapshotCounter;
-   lastSnapshotPrefix = "sid_" + lastSnapshotCounter;
-   LOG.warn("done :: 1");
+  public MutableGraph<CompactionNode> getCompactionReverseDAG() {
+    return compactionDAGReverse;
   }
 
+  /**
+   * Populate the compaction DAG with input and output SST files lists.
+   */
+  @SuppressFBWarnings({"AT_OPERATION_SEQUENCE_ON_CONCURRENT_ABSTRACTION"})
+  private void populateCompactionDAG(List<String> inputFiles,
+      List<String> outputFiles, long seqNum) {
 
-  public void printAllSnapshots() throws InterruptedException {
-    for (Snapshot snap : allSnapshots) {
-      if (snap == null) {
-        break;
-      }
-      LOG.warn("Snapshot id" + snap.snapshotID);
-      LOG.warn("Snapshot path" + snap.dbPath);
-      LOG.warn("Snapshot Generation" + snap.snapshotGeneration);
-      LOG.warn("");
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Input {} -> Output {}", inputFiles, outputFiles);
     }
-  }
 
-  public void diffAllSnapshots() throws InterruptedException, RocksDBException {
-    for (Snapshot snap : allSnapshots) {
-      if (snap == null) {
-        break;
+    for (String outfile : outputFiles) {
+      CompactionNode outfileNode = compactionNodeTable.get(outfile);
+      if (outfileNode == null) {
+        long numKeys = 0L;
+        try {
+          numKeys = getSSTFileSummary(outfile);
+        } catch (Exception e) {
+          LOG.warn("Exception in getSSTFileSummary: {}", e.getMessage());
+        }
+        outfileNode = new CompactionNode(outfile, null, numKeys, seqNum);
+        compactionDAGFwd.addNode(outfileNode);
+        compactionDAGReverse.addNode(outfileNode);
+        compactionNodeTable.put(outfile, outfileNode);
       }
-      printSnapdiffSSTFiles(allSnapshots[lastSnapshotCounter - 1], snap);
-    }
-  }
 
-  public MutableGraph<CompactionNode> getCompactionFwdDAG() {
-    return compactionDAGFwd;
-  }
+      for (String infile : inputFiles) {
+        CompactionNode infileNode = compactionNodeTable.get(infile);

Review Comment:
   Code, from line 898-910 and 913-925, is same. You can extract out the common code to helper function.
   
   ```
     public CompactionNode getCompactionNode(String file) {
       CompactionNode fileNode = compactionNodeTable.get(file);
       if (fileNode != null) {
   
         long numKeys = 0L;
         try {
           numKeys = getSSTFileSummary(outfile);
         } catch (Exception e) {
           LOG.warn("Exception in getSSTFileSummary: {}", e.getMessage());
         }
   
         fileNode = new CompactionNode(file, null, numKeys, seqNum);
         compactionDAGFwd.addNode(fileNode);
         compactionDAGReverse.addNode(fileNode);
         compactionNodeTable.put(file, fileNode);
       }
       return fileNode;
     }
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org