You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2025/02/01 21:12:59 UTC

(accumulo) branch 3.1 updated (5e0455a08b -> 8386e60353)

This is an automated email from the ASF dual-hosted git repository.

kturner pushed a change to branch 3.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git


    from 5e0455a08b Merge branch '2.1' into 3.1
     add 7a27d40463 Offers new ways of computing bulk load plans (#4898)
     new 8386e60353 Merge branch '2.1' into 3.1

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../java/org/apache/accumulo/core/Constants.java   |   1 +
 .../core/client/rfile/LoadPlanCollector.java       | 131 +++++++++++
 .../apache/accumulo/core/client/rfile/RFile.java   |  10 +
 .../accumulo/core/client/rfile/RFileWriter.java    |  36 +++-
 .../core/client/rfile/RFileWriterBuilder.java      |  30 ++-
 .../accumulo/core/clientImpl/bulk/BulkImport.java  |  39 +++-
 .../org/apache/accumulo/core/data/LoadPlan.java    | 240 +++++++++++++++++++++
 .../apache/accumulo/core/file/FileOperations.java  |   5 +
 .../core/client/rfile/RFileClientTest.java         | 210 ++++++++++++++++++
 .../apache/accumulo/core/data/LoadPlanTest.java    |  48 +++++
 .../apache/accumulo/core/file/rfile/RFileTest.java |   1 -
 .../apache/accumulo/test/functional/BulkNewIT.java |  61 +++++-
 12 files changed, 788 insertions(+), 24 deletions(-)
 create mode 100644 core/src/main/java/org/apache/accumulo/core/client/rfile/LoadPlanCollector.java


(accumulo) 01/01: Merge branch '2.1' into 3.1

Posted by kt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch 3.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 8386e60353644fec80866adca605a2f1fea5a759
Merge: 5e0455a08b 7a27d40463
Author: Keith Turner <kt...@apache.org>
AuthorDate: Sat Feb 1 21:12:50 2025 +0000

    Merge branch '2.1' into 3.1

 .../java/org/apache/accumulo/core/Constants.java   |   1 +
 .../core/client/rfile/LoadPlanCollector.java       | 131 +++++++++++
 .../apache/accumulo/core/client/rfile/RFile.java   |  10 +
 .../accumulo/core/client/rfile/RFileWriter.java    |  36 +++-
 .../core/client/rfile/RFileWriterBuilder.java      |  30 ++-
 .../accumulo/core/clientImpl/bulk/BulkImport.java  |  39 +++-
 .../org/apache/accumulo/core/data/LoadPlan.java    | 240 +++++++++++++++++++++
 .../apache/accumulo/core/file/FileOperations.java  |   5 +
 .../core/client/rfile/RFileClientTest.java         | 210 ++++++++++++++++++
 .../apache/accumulo/core/data/LoadPlanTest.java    |  48 +++++
 .../apache/accumulo/core/file/rfile/RFileTest.java |   1 -
 .../apache/accumulo/test/functional/BulkNewIT.java |  61 +++++-
 12 files changed, 788 insertions(+), 24 deletions(-)

diff --cc core/src/main/java/org/apache/accumulo/core/client/rfile/RFile.java
index 64956dcbc2,71c186f3eb..897e87213e
--- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFile.java
@@@ -35,11 -34,10 +35,12 @@@ import org.apache.accumulo.core.client.
  import org.apache.accumulo.core.client.summary.Summary.FileStatistics;
  import org.apache.accumulo.core.conf.Property;
  import org.apache.accumulo.core.data.Key;
+ import org.apache.accumulo.core.data.LoadPlan;
  import org.apache.accumulo.core.data.Range;
  import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.core.util.RowRangeUtil;
  import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.io.Text;
  
  /**
diff --cc core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java
index 559f43df1e,5b12f4d73b..ad5a7e5341
--- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java
@@@ -37,8 -37,8 +37,9 @@@ import org.apache.accumulo.core.conf.Ac
  import org.apache.accumulo.core.conf.ConfigurationCopy;
  import org.apache.accumulo.core.conf.DefaultConfiguration;
  import org.apache.accumulo.core.crypto.CryptoFactoryLoader;
+ import org.apache.accumulo.core.data.LoadPlan;
  import org.apache.accumulo.core.file.FileOperations;
 +import org.apache.accumulo.core.metadata.UnreferencedTabletFile;
  import org.apache.accumulo.core.metadata.ValidationUtil;
  import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
  import org.apache.accumulo.core.spi.crypto.CryptoEnvironment;
@@@ -117,12 -121,13 +122,13 @@@ class RFileWriterBuilder implements RFi
        return new RFileWriter(
            fileops.newWriterBuilder().forOutputStream(".rf", fsdo, out.getConf(), cs)
                .withTableConfiguration(acuconf).withStartDisabled().build(),
-           visCacheSize);
+           visCacheSize, loadPlanCollector);
      } else {
 -      return new RFileWriter(
 -          fileops.newWriterBuilder()
 -              .forFile(out.path.toString(), out.getFileSystem(out.path), out.getConf(), cs)
 -              .withTableConfiguration(acuconf).withStartDisabled().build(),
 -          visCacheSize, loadPlanCollector);
 +      return new RFileWriter(fileops.newWriterBuilder()
 +          .forFile(UnreferencedTabletFile.of(out.getFileSystem(out.path), out.path),
 +              out.getFileSystem(out.path), out.getConf(), cs)
-           .withTableConfiguration(acuconf).withStartDisabled().build(), visCacheSize);
++          .withTableConfiguration(acuconf).withStartDisabled().build(), visCacheSize,
++          loadPlanCollector);
      }
    }
  
diff --cc core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkImport.java
index a5e7ff94b5,1c155cd510..6c9e49886b
--- a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkImport.java
+++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/BulkImport.java
@@@ -345,12 -363,23 +352,22 @@@ public class BulkImport implements Impo
    }
  
    public static List<KeyExtent> findOverlappingTablets(ClientContext context,
-       KeyExtentCache extentCache, UnreferencedTabletFile file, FileSystem fs,
 -      KeyExtentCache keyExtentCache, Path file, FileSystem fs, Cache<String,Long> fileLenCache,
 -      CryptoService cs) throws IOException {
++      KeyExtentCache keyExtentCache, UnreferencedTabletFile file, FileSystem fs,
 +      Cache<String,Long> fileLenCache, CryptoService cs) throws IOException {
      try (FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder()
 -        .forFile(file.toString(), fs, fs.getConf(), cs)
 -        .withTableConfiguration(context.getConfiguration()).withFileLenCache(fileLenCache)
 -        .seekToBeginning().build()) {
 +        .forFile(file, fs, fs.getConf(), cs).withTableConfiguration(context.getConfiguration())
 +        .withFileLenCache(fileLenCache).seekToBeginning().build()) {
-       return findOverlappingTablets(extentCache, reader);
+ 
+       Collection<ByteSequence> columnFamilies = Collections.emptyList();
+       NextRowFunction nextRowFunction = row -> {
+         reader.seek(new Range(row, null), columnFamilies, false);
+         if (!reader.hasTop()) {
+           return null;
+         }
+         return reader.getTopKey().getRow();
+       };
+ 
+       return findOverlappingTablets(keyExtentCache::lookup, nextRowFunction);
      }
    }
  
diff --cc core/src/test/java/org/apache/accumulo/core/client/rfile/RFileClientTest.java
index 753ff6631d,58e56abf0a..4273381467
--- a/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileClientTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileClientTest.java
@@@ -29,6 -28,8 +29,7 @@@ import static org.junit.jupiter.api.Ass
  import java.io.File;
  import java.io.IOException;
  import java.net.ConnectException;
+ import java.net.URI;
 -import java.security.SecureRandom;
  import java.util.AbstractMap;
  import java.util.ArrayList;
  import java.util.Arrays;
@@@ -58,7 -62,8 +63,9 @@@ import org.apache.accumulo.core.conf.Pr
  import org.apache.accumulo.core.data.ArrayByteSequence;
  import org.apache.accumulo.core.data.ByteSequence;
  import org.apache.accumulo.core.data.Key;
+ import org.apache.accumulo.core.data.LoadPlan;
+ import org.apache.accumulo.core.data.LoadPlanTest;
 +import org.apache.accumulo.core.data.PartialKey;
  import org.apache.accumulo.core.data.Range;
  import org.apache.accumulo.core.data.Value;
  import org.apache.accumulo.core.file.FileOperations;