You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by cs...@apache.org on 2023/06/02 23:53:32 UTC
[accumulo] branch no-chop-merge updated (8d39c8e34b -> ba0c7efc53)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a change to branch no-chop-merge
in repository https://gitbox.apache.org/repos/asf/accumulo.git
from 8d39c8e34b Add an optional range to Tablet files and fenced reader (#3418)
add 23bca6905c Rename TabletFile to ReferencedTabletFile (#3449)
new f24f8ac050 Merge branch 'main' into no-chop-merge
add 96814b81f9 Add a new TabletFile interface (#3450)
new ba0c7efc53 Merge branch 'main' into no-chop-merge
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../accumulo/core/file/BloomFilterLayer.java | 6 +-
.../accumulo/core/file/DispatchingFileFactory.java | 4 +-
.../apache/accumulo/core/file/FileOperations.java | 26 ++--
.../org/apache/accumulo/core/file/rfile/RFile.java | 6 +-
.../accumulo/core/file/rfile/RFileOperations.java | 4 +-
.../apache/accumulo/core/logging/TabletLogger.java | 4 +-
.../accumulo/core/metadata/AbstractTabletFile.java | 21 +---
.../{TabletFile.java => ReferencedTabletFile.java} | 23 ++--
.../core/metadata/ScanServerRefTabletFile.java | 2 +-
.../accumulo/core/metadata/StoredTabletFile.java | 12 +-
.../apache/accumulo/core/metadata/TabletFile.java | 134 ++-------------------
.../core/metadata/UnreferencedTabletFile.java | 4 +-
.../accumulo/core/metadata/schema/Ample.java | 10 +-
.../schema/ExternalCompactionMetadata.java | 15 +--
...FileTest.java => ReferencedTabletFileTest.java} | 14 +--
.../accumulo/server/compaction/FileCompactor.java | 10 +-
.../org/apache/accumulo/server/fs/VolumeUtil.java | 6 +-
.../server/init/FileSystemInitializer.java | 4 +-
.../server/metadata/TabletMutatorBase.java | 8 +-
.../org/apache/accumulo/server/util/FileUtil.java | 38 +++---
.../accumulo/server/util/ManagerMetadataUtil.java | 11 +-
.../accumulo/server/util/MetadataTableUtil.java | 7 +-
.../accumulo/server/util/TableDiskUsageTest.java | 12 +-
.../org/apache/accumulo/compactor/Compactor.java | 5 +-
.../manager/tableOps/bulkVer2/LoadFiles.java | 6 +-
.../accumulo/tserver/TabletClientHandler.java | 10 +-
.../tserver/compactions/ExternalCompactionJob.java | 6 +-
.../accumulo/tserver/tablet/CompactableImpl.java | 7 +-
.../accumulo/tserver/tablet/CompactableUtils.java | 13 +-
.../accumulo/tserver/tablet/DatafileManager.java | 17 +--
.../tserver/tablet/MinorCompactionTask.java | 9 +-
.../accumulo/tserver/tablet/MinorCompactor.java | 5 +-
.../org/apache/accumulo/tserver/tablet/Tablet.java | 39 +++---
.../apache/accumulo/tserver/tablet/TabletData.java | 8 +-
.../tserver/compaction/CompactableUtilsTest.java | 9 +-
.../test/functional/GarbageCollectorTrashBase.java | 4 +-
.../accumulo/test/functional/SplitRecoveryIT.java | 18 +--
.../test/performance/scan/CollectTabletStats.java | 6 +-
38 files changed, 220 insertions(+), 323 deletions(-)
copy core/src/main/java/org/apache/accumulo/core/metadata/{TabletFile.java => ReferencedTabletFile.java} (86%)
rename core/src/test/java/org/apache/accumulo/core/metadata/schema/{TabletFileTest.java => ReferencedTabletFileTest.java} (91%)
[accumulo] 02/02: Merge branch 'main' into no-chop-merge
Posted by cs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch no-chop-merge
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit ba0c7efc53588f89bfc484efde5b27e4cdd58519
Merge: f24f8ac050 96814b81f9
Author: Christopher L. Shannon (cshannon) <ch...@gmail.com>
AuthorDate: Fri Jun 2 19:48:20 2023 -0400
Merge branch 'main' into no-chop-merge
.../accumulo/core/file/DispatchingFileFactory.java | 4 +--
.../apache/accumulo/core/file/FileOperations.java | 26 +++++++--------
.../org/apache/accumulo/core/file/rfile/RFile.java | 6 ++--
.../accumulo/core/file/rfile/RFileOperations.java | 4 +--
.../apache/accumulo/core/logging/TabletLogger.java | 4 +--
.../accumulo/core/metadata/AbstractTabletFile.java | 21 ++++--------
.../{AbstractTabletFile.java => TabletFile.java} | 37 +++-------------------
.../org/apache/accumulo/server/util/FileUtil.java | 33 ++++++++++---------
.../test/performance/scan/CollectTabletStats.java | 6 ++--
9 files changed, 52 insertions(+), 89 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
index a0beb6ef13,7618777892..21c5d51a9d
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java
@@@ -74,7 -71,6 +74,7 @@@ import org.apache.accumulo.core.iterato
import org.apache.accumulo.core.iteratorsImpl.system.LocalityGroupIterator.LocalityGroup;
import org.apache.accumulo.core.iteratorsImpl.system.LocalityGroupIterator.LocalityGroupContext;
import org.apache.accumulo.core.iteratorsImpl.system.LocalityGroupIterator.LocalityGroupSeekCache;
- import org.apache.accumulo.core.metadata.AbstractTabletFile;
++import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
import org.apache.accumulo.core.util.LocalityGroupUtil;
import org.apache.accumulo.core.util.MutableByteSequence;
@@@ -1558,208 -1553,5 +1558,208 @@@ public class RFile
public void setCacheProvider(CacheProvider cacheProvider) {
reader.setCacheProvider(cacheProvider);
}
+
+ @Override
+ public void reset() {
+ clear();
+ }
+ }
+
+ public interface RFileSKVIterator extends FileSKVIterator {
+ FileSKVIterator getIndex() throws IOException;
+
+ void reset();
+ }
+
+ static abstract class FencedFileSKVIterator implements FileSKVIterator {
+
+ private final FileSKVIterator reader;
+ protected final Range fence;
+
+ public FencedFileSKVIterator(FileSKVIterator reader, Range fence) {
+ this.reader = Objects.requireNonNull(reader);
+ this.fence = fence;
+ }
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options,
+ IteratorEnvironment env) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean hasTop() {
+ return reader.hasTop();
+ }
+
+ @Override
+ public void next() throws IOException {
+ reader.next();
+ }
+
+ @Override
+ public Key getTopKey() {
+ return reader.getTopKey();
+ }
+
+ @Override
+ public Value getTopValue() {
+ return reader.getTopValue();
+ }
+
+ @Override
+ public Key getFirstKey() throws IOException {
+ var rfk = reader.getFirstKey();
+ if (fence.beforeStartKey(rfk)) {
+ return fence.getStartKey();
+ } else {
+ return rfk;
+ }
+ }
+
+ @Override
+ public Key getLastKey() throws IOException {
+ var rlk = reader.getLastKey();
+ if (fence.afterEndKey(rlk)) {
+ return fence.getEndKey();
+ } else {
+ return rlk;
+ }
+ }
+
+ @Override
+ public boolean isRunningLowOnMemory() {
+ return reader.isRunningLowOnMemory();
+ }
+
+ @Override
+ public void setInterruptFlag(AtomicBoolean flag) {
+ reader.setInterruptFlag(flag);
+ }
+
+ @Override
+ public DataInputStream getMetaStore(String name) throws IOException {
+ return reader.getMetaStore(name);
+ }
+
+ @Override
+ public void closeDeepCopies() throws IOException {
+ reader.closeDeepCopies();
+ }
+
+ @Override
+ public void setCacheProvider(CacheProvider cacheProvider) {
+ reader.setCacheProvider(cacheProvider);
+ }
+
+ @Override
+ public void close() throws IOException {
+ reader.close();
+ }
+ }
+
+ static class FencedIndex extends FencedFileSKVIterator {
+ private final FileSKVIterator source;
+
+ public FencedIndex(FileSKVIterator source, Range seekFence) {
+ super(source, seekFence);
+ this.source = source;
+ }
+
+ @Override
+ public boolean hasTop() {
+ // this code filters out data because the rfile index iterators do not support seek
+
+ // If startKey is set then discard everything until we reach the start
+ // of the range
+ if (fence.getStartKey() != null) {
+
+ while (source.hasTop() && fence.beforeStartKey(source.getTopKey())) {
+ try {
+ source.next();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+ }
+
+ // If endKey is set then ensure that the current key is not passed the end of the range
+ return source.hasTop() && !fence.afterEndKey(source.getTopKey());
+ }
+
+ @Override
+ public FileSKVIterator getSample(SamplerConfigurationImpl sampleConfig) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive)
+ throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ static class FencedReader extends FencedFileSKVIterator implements RFileSKVIterator {
+
+ private final Reader reader;
+
+ public FencedReader(Reader reader, Range seekFence) {
+ super(reader, seekFence);
+ this.reader = reader;
+ }
+
+ @Override
+ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive)
+ throws IOException {
+ reader.reset();
+
+ if (fence != null) {
+ range = fence.clip(range, true);
+ if (range == null) {
+ return;
+ }
+ }
+
+ reader.seek(range, columnFamilies, inclusive);
+ }
+
+ @Override
+ public FencedReader deepCopy(IteratorEnvironment env) {
+ return new FencedReader(reader.deepCopy(env), fence);
+ }
+
+ @Override
+ public FileSKVIterator getIndex() throws IOException {
+ return new FencedIndex(reader.getIndex(), fence);
+ }
+
+ @Override
+ public FileSKVIterator getSample(SamplerConfigurationImpl sampleConfig) {
+ final Reader sample = reader.getSample(sampleConfig);
+ return sample != null ? new FencedReader(sample, fence) : null;
+ }
+
+ @Override
+ public void reset() {
+ reader.reset();
+ }
+ }
+
- public static RFileSKVIterator getReader(final CachableBuilder cb,
- final AbstractTabletFile<?> dataFile) throws IOException {
++ public static RFileSKVIterator getReader(final CachableBuilder cb, final TabletFile dataFile)
++ throws IOException {
+ final RFile.Reader reader = new RFile.Reader(Objects.requireNonNull(cb));
+ return dataFile.hasRange() ? new FencedReader(reader, dataFile.getRange()) : reader;
+ }
+
+ public static RFileSKVIterator getReader(final CachableBuilder cb, Range range)
+ throws IOException {
+ final RFile.Reader reader = new RFile.Reader(Objects.requireNonNull(cb));
+ return !Objects.requireNonNull(range).isInfiniteStartKey() || !range.isInfiniteStopKey()
+ ? new FencedReader(reader, range) : reader;
}
}
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/RFileOperations.java
index 2d79e13e99,e2f62654a7..cf3e9f64e2
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFileOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFileOperations.java
@@@ -33,9 -33,8 +33,9 @@@ import org.apache.accumulo.core.file.Fi
import org.apache.accumulo.core.file.FileSKVIterator;
import org.apache.accumulo.core.file.FileSKVWriter;
import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile.CachableBuilder;
+import org.apache.accumulo.core.file.rfile.RFile.RFileSKVIterator;
import org.apache.accumulo.core.file.rfile.bcfile.BCFile;
- import org.apache.accumulo.core.metadata.AbstractTabletFile;
+ import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
import org.apache.accumulo.core.sample.impl.SamplerFactory;
import org.apache.hadoop.conf.Configuration;
diff --cc core/src/main/java/org/apache/accumulo/core/metadata/AbstractTabletFile.java
index 546b40d73c,587636c207..ef01a7882a
--- a/core/src/main/java/org/apache/accumulo/core/metadata/AbstractTabletFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/AbstractTabletFile.java
@@@ -56,19 -50,4 +53,13 @@@ public abstract class AbstractTabletFil
return path;
}
- /**
- * @return The range of the TabletFile
- *
- */
++ @Override
+ public Range getRange() {
+ return range;
+ }
+
- /**
- * @return True if this file is fenced by a range
- *
- */
++ @Override
+ public boolean hasRange() {
+ return !range.isInfiniteStartKey() || !range.isInfiniteStopKey();
+ }
}
diff --cc core/src/main/java/org/apache/accumulo/core/metadata/TabletFile.java
index 546b40d73c,baad4f5103..14dd2de5ec
--- a/core/src/main/java/org/apache/accumulo/core/metadata/TabletFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/TabletFile.java
@@@ -18,29 -18,9 +18,10 @@@
*/
package org.apache.accumulo.core.metadata;
- import java.util.Objects;
-
+import org.apache.accumulo.core.data.Range;
import org.apache.hadoop.fs.Path;
- /**
- * A base class used to represent file references that are handled by code that processes tablet
- * files.
- *
- * @since 3.0.0
- */
- public abstract class AbstractTabletFile<T extends AbstractTabletFile<T>> implements Comparable<T> {
-
- private final String fileName; // C0004.rf
- protected final Path path;
- protected final Range range;
-
- protected AbstractTabletFile(Path path, Range range) {
- this.path = Objects.requireNonNull(path);
- this.fileName = path.getName();
- this.range = Objects.requireNonNull(range);
- ValidationUtil.validateFileName(fileName);
- }
+ public interface TabletFile {
/**
* @return The file name of the TabletFile
@@@ -52,23 -30,5 +31,17 @@@
/**
* @return The path of the TabletFile
*/
- public Path getPath() {
- return path;
- }
+ Path getPath();
+
+ /**
+ * @return The range of the TabletFile
+ *
+ */
- public Range getRange() {
- return range;
- }
++ Range getRange();
+
+ /**
+ * @return True if this file is fenced by a range
+ *
+ */
- public boolean hasRange() {
- return !range.isInfiniteStartKey() || !range.isInfiniteStopKey();
- }
++ boolean hasRange();
}
[accumulo] 01/02: Merge branch 'main' into no-chop-merge
Posted by cs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch no-chop-merge
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit f24f8ac050cc9fac4d9e566449b6117493043f96
Merge: 8d39c8e34b 23bca6905c
Author: Christopher L. Shannon (cshannon) <ch...@gmail.com>
AuthorDate: Fri Jun 2 19:19:48 2023 -0400
Merge branch 'main' into no-chop-merge
.../accumulo/core/file/BloomFilterLayer.java | 6 ++--
.../{TabletFile.java => ReferencedTabletFile.java} | 23 +++++++------
.../core/metadata/ScanServerRefTabletFile.java | 2 +-
.../accumulo/core/metadata/StoredTabletFile.java | 12 +++----
.../core/metadata/UnreferencedTabletFile.java | 4 +--
.../accumulo/core/metadata/schema/Ample.java | 10 +++---
.../schema/ExternalCompactionMetadata.java | 15 +++++----
...FileTest.java => ReferencedTabletFileTest.java} | 14 ++++----
.../accumulo/server/compaction/FileCompactor.java | 10 +++---
.../org/apache/accumulo/server/fs/VolumeUtil.java | 6 ++--
.../server/init/FileSystemInitializer.java | 4 +--
.../server/metadata/TabletMutatorBase.java | 8 ++---
.../org/apache/accumulo/server/util/FileUtil.java | 7 ++--
.../accumulo/server/util/ManagerMetadataUtil.java | 11 +++---
.../accumulo/server/util/MetadataTableUtil.java | 7 ++--
.../accumulo/server/util/TableDiskUsageTest.java | 12 ++++---
.../org/apache/accumulo/compactor/Compactor.java | 5 +--
.../manager/tableOps/bulkVer2/LoadFiles.java | 6 ++--
.../accumulo/tserver/TabletClientHandler.java | 10 +++---
.../tserver/compactions/ExternalCompactionJob.java | 6 ++--
.../accumulo/tserver/tablet/CompactableImpl.java | 7 ++--
.../accumulo/tserver/tablet/CompactableUtils.java | 13 ++++----
.../accumulo/tserver/tablet/DatafileManager.java | 17 +++++-----
.../tserver/tablet/MinorCompactionTask.java | 9 ++---
.../accumulo/tserver/tablet/MinorCompactor.java | 5 +--
.../org/apache/accumulo/tserver/tablet/Tablet.java | 39 ++++++++++++----------
.../apache/accumulo/tserver/tablet/TabletData.java | 8 ++---
.../tserver/compaction/CompactableUtilsTest.java | 9 ++---
.../test/functional/GarbageCollectorTrashBase.java | 4 +--
.../accumulo/test/functional/SplitRecoveryIT.java | 18 +++++-----
30 files changed, 163 insertions(+), 144 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java
index 52cd13dc49,abda1e5672..70847d6ea3
--- a/core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java
@@@ -47,21 -45,14 +47,22 @@@ public class ReferencedTabletFile exten
private final TabletDirectory tabletDir; // hdfs://1.2.3.4/accumulo/tables/2a/t-0003
private final String normalizedPath;
- private static final Logger log = LoggerFactory.getLogger(TabletFile.class);
+ private static final Logger log = LoggerFactory.getLogger(ReferencedTabletFile.class);
- private static final Comparator<TabletFile> comparator =
- Comparator.comparing(TabletFile::getNormalizedPathStr).thenComparing(TabletFile::getRange);
++ private static final Comparator<ReferencedTabletFile> comparator =
++ Comparator.comparing(ReferencedTabletFile::getNormalizedPathStr)
++ .thenComparing(ReferencedTabletFile::getRange);
+
- public TabletFile(Path metaPath) {
++ public ReferencedTabletFile(Path metaPath) {
+ this(metaPath, new Range());
+ }
+
/**
* Construct new tablet file using a Path. Used in the case where we had to use Path object to
* qualify an absolute path or create a new file.
*/
- public TabletFile(Path metaPath, Range range) {
- public ReferencedTabletFile(Path metaPath) {
- super(Objects.requireNonNull(metaPath));
++ public ReferencedTabletFile(Path metaPath, Range range) {
+ super(Objects.requireNonNull(metaPath), range);
String errorMsg = "Missing or invalid part of tablet file metadata entry: " + metaPath;
log.trace("Parsing TabletFile from {}", metaPath);
@@@ -135,9 -126,9 +136,9 @@@
@Override
public boolean equals(Object obj) {
- if (obj instanceof TabletFile) {
- TabletFile that = (TabletFile) obj;
+ if (obj instanceof ReferencedTabletFile) {
+ ReferencedTabletFile that = (ReferencedTabletFile) obj;
- return normalizedPath.equals(that.normalizedPath);
+ return normalizedPath.equals(that.normalizedPath) && range.equals(that.range);
}
return false;
}
diff --cc core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
index e8f3b2ce9d,4b517360d1..0477e0098a
--- a/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
@@@ -45,11 -44,9 +45,11 @@@ public class StoredTabletFile extends A
* the entry can be deleted.
*/
public StoredTabletFile(String metadataEntry) {
- super(new Path(metadataEntry));
+ // TODO: Future version of metadataEntry will contains the path
+ // and the range so we will need to parse the string here
+ super(new Path(metadataEntry), new Range());
this.metadataEntry = metadataEntry;
- this.tabletFile = TabletFile.of(getPath());
+ this.referencedTabletFile = ReferencedTabletFile.of(getPath());
}
/**