[GitHub] [accumulo] milleruntime commented on a change in pull request #1367: Retry new Bulk import on merge. Fixes #471

[GitBox <gi...@apache.org>]

milleruntime commented on a change in pull request #1367: Retry new Bulk import on merge. Fixes #471
URL: https://github.com/apache/accumulo/pull/1367#discussion_r332083134
 
 

 ##########
 File path: core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkImport.java
 ##########
 @@ -125,21 +126,47 @@ public void load()
     Path srcPath = checkPath(fs, dir);
 
     SortedMap<KeyExtent,Bulk.Files> mappings;
-    if (plan == null) {
-      mappings = computeMappingFromFiles(fs, tableId, srcPath);
-    } else {
-      mappings = computeMappingFromPlan(fs, tableId, srcPath);
-    }
+    TableOperationsImpl tableOps = new TableOperationsImpl(context);
+    // retry if a merge occurs
+    boolean retry = true;
+    while (retry) {
+      if (plan == null) {
+        mappings = computeMappingFromFiles(fs, tableId, srcPath);
+      } else {
+        mappings = computeMappingFromPlan(fs, tableId, srcPath);
+      }
 
-    if (mappings.isEmpty())
-      throw new IllegalArgumentException("Attempted to import zero files from " + srcPath);
+      if (mappings.isEmpty())
+        throw new IllegalArgumentException("Attempted to import zero files from " + srcPath);
 
-    BulkSerialize.writeLoadMapping(mappings, srcPath.toString(), fs::create);
+      BulkSerialize.writeLoadMapping(mappings, srcPath.toString(), fs::create);
+
+      List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.canonical().getBytes(UTF_8)),
+          ByteBuffer.wrap(srcPath.toString().getBytes(UTF_8)),
+          ByteBuffer.wrap((setTime + "").getBytes(UTF_8)));
+      try {
+        tableOps.doBulkFateOperation(args, tableName);
+        retry = false;
+      } catch (AccumuloBulkMergeException ae) {
+        if (plan != null) {
+          checkPlanForSplits();
+        }
+        log.info(ae.getMessage() + ". Retrying bulk import to " + tableName);
+      }
+    }
+  }
 
-    List<ByteBuffer> args = Arrays.asList(ByteBuffer.wrap(tableId.canonical().getBytes(UTF_8)),
-        ByteBuffer.wrap(srcPath.toString().getBytes(UTF_8)),
-        ByteBuffer.wrap((setTime + "").getBytes(UTF_8)));
-    new TableOperationsImpl(context).doBulkFateOperation(args, tableName);
+  /**
+   * Check if splits were specified in plan when a concurrent merge occurred. If so, throw error
+   * back to user since retrying won't help. If not, then retry.
+   */
+  private void checkPlanForSplits() throws AccumuloException {
+    for (Destination des : plan.getDestinations()) {
+      if (des.getRangeType().equals(RangeType.TABLE)) {
+        throw new AccumuloException(
+            "The splits provided in Load Plan do not exist in " + tableName);
 
 Review comment:
   I was thinking it wouldn't be useful to the user but I guess it wouldn't hurt to include the AccumuloBulkMergeException

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services