You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2023/05/12 04:29:38 UTC
[accumulo] 01/01: Merge branch '1.10' into 2.1
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 11320ee8e6906265f9c10e0974322a6665f02df4
Merge: 1fbac624e4 3cc35be037
Author: Christopher Tubbs <ct...@apache.org>
AuthorDate: Fri May 12 00:09:20 2023 -0400
Merge branch '1.10' into 2.1
.../org/apache/accumulo/core/conf/Property.java | 2 +-
.../apache/accumulo/core/conf/PropertyType.java | 7 +-
.../core/iteratorsImpl/system/MapFileIterator.java | 109 ++++++---------------
.../accumulo/core/conf/PropertyTypeTest.java | 7 ++
4 files changed, 42 insertions(+), 83 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 3417cea934,369e310489..a47a0992ac
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -1018,92 -682,41 +1018,92 @@@ public enum Property
TABLE_MINC_COMPACT_IDLETIME("table.compaction.minor.idle", "5m", PropertyType.TIMEDURATION,
"After a tablet has been idle (no mutations) for this time period it may have its "
+ "in-memory map flushed to disk in a minor compaction. There is no guarantee an idle "
- + "tablet will be compacted."),
- TABLE_MINC_MAX_MERGE_FILE_SIZE("table.compaction.minor.merge.file.size.max", "1",
- PropertyType.MEMORY,
- "The max file size used for a merging minor compaction. The value of 0 is no max "
- + "file size. The default value is 1 byte, which should disable merging minor compactions."),
- TABLE_SCAN_MAXMEM("table.scan.max.memory", "512K", PropertyType.MEMORY,
+ + "tablet will be compacted.",
+ "1.3.5"),
+ TABLE_COMPACTION_DISPATCHER("table.compaction.dispatcher",
+ SimpleCompactionDispatcher.class.getName(), PropertyType.CLASSNAME,
+ "A configurable dispatcher that decides what compaction service a table should use.",
+ "2.1.0"),
+ TABLE_COMPACTION_DISPATCHER_OPTS("table.compaction.dispatcher.opts.", null, PropertyType.PREFIX,
+ "Options for the table compaction dispatcher", "2.1.0"),
+ TABLE_COMPACTION_SELECTION_EXPIRATION("table.compaction.selection.expiration.ms", "2m",
+ PropertyType.TIMEDURATION,
+ "User compactions select files and are then queued for compaction, preventing these files "
+ + "from being used in system compactions. This timeout allows system compactions to cancel "
+ + "the hold queued user compactions have on files, when its queued for more than the "
+ + "specified time. If a system compaction cancels a hold and runs, then the user compaction"
+ + " can reselect and hold files after the system compaction runs.",
+ "2.1.0"),
+ TABLE_COMPACTION_SELECTOR("table.compaction.selector", "", PropertyType.CLASSNAME,
+ "A configurable selector for a table that can periodically select file for mandatory "
+ + "compaction, even if the files do not meet the compaction ratio.",
+ "2.1.0"),
+ TABLE_COMPACTION_SELECTOR_OPTS("table.compaction.selector.opts.", null, PropertyType.PREFIX,
+ "Options for the table compaction dispatcher", "2.1.0"),
+ TABLE_COMPACTION_CONFIGURER("table.compaction.configurer", "", PropertyType.CLASSNAME,
+ "A plugin that can dynamically configure compaction output files based on input files.",
+ "2.1.0"),
+ TABLE_COMPACTION_CONFIGURER_OPTS("table.compaction.configurer.opts.", null, PropertyType.PREFIX,
+ "Options for the table compaction configuror", "2.1.0"),
+ @Deprecated(since = "2.1.0", forRemoval = true)
+ @ReplacedBy(property = TABLE_COMPACTION_SELECTOR)
+ TABLE_COMPACTION_STRATEGY("table.majc.compaction.strategy",
+ "org.apache.accumulo.tserver.compaction.DefaultCompactionStrategy", PropertyType.CLASSNAME,
+ "See {% jlink -f org.apache.accumulo.core.spi.compaction}", "1.6.0"),
+ @Deprecated(since = "2.1.0", forRemoval = true)
+ @ReplacedBy(property = TABLE_COMPACTION_SELECTOR_OPTS)
+ TABLE_COMPACTION_STRATEGY_PREFIX("table.majc.compaction.strategy.opts.", null,
+ PropertyType.PREFIX,
+ "Properties in this category are used to configure the compaction strategy.", "1.6.0"),
+ // Crypto-related properties
+ @Experimental
+ TABLE_CRYPTO_PREFIX("table.crypto.opts.", null, PropertyType.PREFIX,
+ "Properties related to on-disk file encryption.", "2.1.0"),
+ @Experimental
+ @Sensitive
+ TABLE_CRYPTO_SENSITIVE_PREFIX("table.crypto.opts.sensitive.", null, PropertyType.PREFIX,
+ "Sensitive properties related to on-disk file encryption.", "2.1.0"),
+ TABLE_SCAN_DISPATCHER("table.scan.dispatcher", SimpleScanDispatcher.class.getName(),
+ PropertyType.CLASSNAME,
+ "This class is used to dynamically dispatch scans to configured scan executors. Configured "
+ + "classes must implement {% jlink " + ScanDispatcher.class.getName() + " %} See "
+ + "[scan executors]({% durl administration/scan-executors %}) for an overview of why"
+ + " and how to use this property. This property is ignored for the root and metadata"
+ + " table. The metadata table always dispatches to a scan executor named `meta`.",
+ "2.0.0"),
+ TABLE_SCAN_DISPATCHER_OPTS("table.scan.dispatcher.opts.", null, PropertyType.PREFIX,
+ "Options for the table scan dispatcher", "2.0.0"),
+ TABLE_SCAN_MAXMEM("table.scan.max.memory", "512k", PropertyType.BYTES,
"The maximum amount of memory that will be used to cache results of a client query/scan. "
- + "Once this limit is reached, the buffered data is sent to the client."),
+ + "Once this limit is reached, the buffered data is sent to the client.",
+ "1.3.5"),
- TABLE_FILE_TYPE("table.file.type", RFile.EXTENSION, PropertyType.STRING,
+ TABLE_FILE_TYPE("table.file.type", RFile.EXTENSION, PropertyType.FILENAME_EXT,
- "Change the type of file a table writes"),
- TABLE_LOAD_BALANCER("table.balancer",
- "org.apache.accumulo.server.master.balancer.DefaultLoadBalancer", PropertyType.STRING,
+ "Change the type of file a table writes", "1.3.5"),
+ TABLE_LOAD_BALANCER("table.balancer", "org.apache.accumulo.core.spi.balancer.SimpleLoadBalancer",
+ PropertyType.STRING,
"This property can be set to allow the LoadBalanceByTable load balancer"
- + " to change the called Load Balancer for this table"),
+ + " to change the called Load Balancer for this table",
+ "1.3.5"),
TABLE_FILE_COMPRESSION_TYPE("table.file.compress.type", "gz", PropertyType.STRING,
- "One of gz,snappy,lzo,none"),
- TABLE_FILE_COMPRESSED_BLOCK_SIZE("table.file.compress.blocksize", "100K", PropertyType.MEMORY,
- "Similar to the hadoop io.seqfile.compress.blocksize setting, so that"
- + " files have better query performance. The maximum value for this is "
- + Integer.MAX_VALUE + ". (This setting is the size threshold prior to"
- + " compression, and applies even compression is disabled.)"),
- TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX("table.file.compress.blocksize.index", "128K",
- PropertyType.MEMORY,
- "Determines how large index blocks can be in files that support"
- + " multilevel indexes. The maximum value for this is " + Integer.MAX_VALUE
- + ". (This setting is the size threshold prior to compression, and applies"
- + " even compression is disabled.)"),
- TABLE_FILE_BLOCK_SIZE("table.file.blocksize", "0B", PropertyType.MEMORY,
- "Overrides the hadoop dfs.block.size setting so that files have better"
- + " query performance. The maximum value for this is " + Integer.MAX_VALUE),
+ "Compression algorithm used on index and data blocks before they are"
+ + " written. Possible values: zstd, gz, snappy, bzip2, lzo, lz4, none",
+ "1.3.5"),
+ TABLE_FILE_COMPRESSED_BLOCK_SIZE("table.file.compress.blocksize", "100k", PropertyType.BYTES,
+ "The maximum size of data blocks in RFiles before they are compressed and written.", "1.3.5"),
+ TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX("table.file.compress.blocksize.index", "128k",
+ PropertyType.BYTES,
+ "The maximum size of index blocks in RFiles before they are compressed and written.",
+ "1.4.0"),
+ TABLE_FILE_BLOCK_SIZE("table.file.blocksize", "0B", PropertyType.BYTES,
+ "The HDFS block size used when writing RFiles. When set to 0B, the"
+ + " value/defaults of HDFS property 'dfs.block.size' will be used.",
+ "1.3.5"),
TABLE_FILE_REPLICATION("table.file.replication", "0", PropertyType.COUNT,
- "Determines how many replicas to keep of a tables' files in HDFS. "
- + "When this value is LTE 0, HDFS defaults are used."),
+ "The number of replicas for a table's RFiles in HDFS. When set to 0, HDFS"
+ + " defaults are used.",
+ "1.3.5"),
TABLE_FILE_MAX("table.file.max", "15", PropertyType.COUNT,
- "Determines the max # of files each tablet in a table can have. When"
+ "The maximum number of RFiles each tablet in a table can have. When"
+ " adjusting this property you may want to consider adjusting"
+ " table.compaction.major.ratio also. Setting this property to 0 will make"
+ " it default to tserver.scan.files.open.max-1, this will prevent a tablet"
diff --cc core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java
index b13a026c8c,f8c468d87c..5d52e58965
--- a/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java
@@@ -26,10 -24,12 +26,11 @@@ import java.util.function.Function
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.IntStream;
import java.util.stream.Stream;
-import org.apache.accumulo.core.Constants;
+ import org.apache.accumulo.core.file.rfile.RFile;
-import org.apache.accumulo.core.util.Pair;
-import org.apache.commons.lang.math.IntRange;
+import org.apache.commons.lang3.Range;
import org.apache.hadoop.fs.Path;
import com.google.common.base.Preconditions;
diff --cc core/src/main/java/org/apache/accumulo/core/iteratorsImpl/system/MapFileIterator.java
index 3fddf87350,0000000000..bc3d317903
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/system/MapFileIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/system/MapFileIterator.java
@@@ -1,174 -1,0 +1,121 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.iteratorsImpl.system;
+
+import java.io.DataInputStream;
- import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileSKVIterator;
- import org.apache.accumulo.core.file.NoSuchMetaStoreException;
+import org.apache.accumulo.core.file.blockfile.impl.CacheProvider;
- import org.apache.accumulo.core.file.map.MapFileUtil;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
- import org.apache.hadoop.fs.Path;
- import org.apache.hadoop.io.MapFile.Reader;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
+
+public class MapFileIterator implements FileSKVIterator {
- private static final Logger log = LoggerFactory.getLogger(MapFileIterator.class);
+
- private Reader reader;
- private Value topValue;
- private Key topKey;
- private AtomicBoolean interruptFlag;
- private int interruptCheckCount = 0;
- private FileSystem fs;
- private String dirName;
++ private static final String MSG = "Map files are not supported";
+
- public MapFileIterator(FileSystem fs, String dir, Configuration conf) throws IOException {
- this.reader = MapFileUtil.openMapFile(fs, dir, conf);
- this.fs = fs;
- this.dirName = dir;
++ public MapFileIterator(FileSystem fs, String dir, Configuration conf) {
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
+ public void setInterruptFlag(AtomicBoolean flag) {
- this.interruptFlag = flag;
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options,
- IteratorEnvironment env) throws IOException {
- throw new UnsupportedOperationException();
++ IteratorEnvironment env) {
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
+ public boolean hasTop() {
- return topKey != null;
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
- public void next() throws IOException {
- if (interruptFlag != null && interruptCheckCount++ % 100 == 0 && interruptFlag.get()) {
- throw new IterationInterruptedException();
- }
-
- reader.next(topKey, topValue);
++ public void next() {
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
- public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive)
- throws IOException {
- if (!columnFamilies.isEmpty() || inclusive) {
- throw new IllegalArgumentException("I do not know how to filter column families");
- }
-
- if (range == null) {
- throw new IllegalArgumentException("Cannot seek to null range");
- }
-
- if (interruptFlag != null && interruptFlag.get()) {
- throw new IterationInterruptedException();
- }
-
- Key key = range.getStartKey();
- if (key == null) {
- key = new Key();
- }
-
- reader.seek(key);
-
- while (hasTop() && range.beforeStartKey(getTopKey())) {
- next();
- }
++ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) {
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
+ public Key getTopKey() {
- return topKey;
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
+ public Value getTopValue() {
- return topValue;
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
- try {
- @SuppressWarnings("deprecation")
- SortedKeyValueIterator<Key,Value> other = env.reserveMapFileReader(dirName);
- ((InterruptibleIterator) other).setInterruptFlag(interruptFlag);
- log.debug("deep copying MapFile: {} -> {}", this, other);
- return other;
- } catch (IOException e) {
- log.error("failed to clone map file reader", e);
- throw new RuntimeException(e);
- }
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
- public Key getFirstKey() throws IOException {
- throw new UnsupportedOperationException();
++ public Key getFirstKey() {
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
- public Key getLastKey() throws IOException {
- throw new UnsupportedOperationException();
++ public Key getLastKey() {
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
- public DataInputStream getMetaStore(String name) throws IOException {
- Path path = new Path(this.dirName, name);
- if (!fs.exists(path)) {
- throw new NoSuchMetaStoreException("name = " + name);
- }
- return fs.open(path);
++ public DataInputStream getMetaStore(String name) {
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
- public void closeDeepCopies() throws IOException {
- // nothing to do, deep copies are externally managed/closed
++ public void closeDeepCopies() {
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
- public void close() throws IOException {
- reader.close();
++ public void close() {
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
+ public FileSKVIterator getSample(SamplerConfigurationImpl sampleConfig) {
- return null;
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
- public void setCacheProvider(CacheProvider cacheProvider) {}
++ public void setCacheProvider(CacheProvider cacheProvider) {
++ throw new UnsupportedOperationException(MSG);
++ }
+}
diff --cc core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java
index 514fd1e95c,53ef7c4003..41a651ebe6
--- a/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java
@@@ -27,9 -25,11 +27,10 @@@ import java.util.Set
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import org.apache.accumulo.core.WithTestNames;
+ import org.apache.accumulo.core.file.rfile.RFile;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
import com.google.common.base.Joiner;