You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2023/05/12 04:29:37 UTC
[accumulo] branch 2.1 updated (1fbac624e4 -> 11320ee8e6)
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a change to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
from 1fbac624e4 Show manager goal state in monitor (#3389)
add 3cc35be037 Disable MapFile in 1.10 and 2.1 patch releases (#3391)
new 11320ee8e6 Merge branch '1.10' into 2.1
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/accumulo/core/conf/Property.java | 2 +-
.../apache/accumulo/core/conf/PropertyType.java | 7 +-
.../core/iteratorsImpl/system/MapFileIterator.java | 109 ++++++---------------
.../accumulo/core/conf/PropertyTypeTest.java | 7 ++
4 files changed, 42 insertions(+), 83 deletions(-)
[accumulo] 01/01: Merge branch '1.10' into 2.1
Posted by ct...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 11320ee8e6906265f9c10e0974322a6665f02df4
Merge: 1fbac624e4 3cc35be037
Author: Christopher Tubbs <ct...@apache.org>
AuthorDate: Fri May 12 00:09:20 2023 -0400
Merge branch '1.10' into 2.1
.../org/apache/accumulo/core/conf/Property.java | 2 +-
.../apache/accumulo/core/conf/PropertyType.java | 7 +-
.../core/iteratorsImpl/system/MapFileIterator.java | 109 ++++++---------------
.../accumulo/core/conf/PropertyTypeTest.java | 7 ++
4 files changed, 42 insertions(+), 83 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 3417cea934,369e310489..a47a0992ac
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -1018,92 -682,41 +1018,92 @@@ public enum Property
TABLE_MINC_COMPACT_IDLETIME("table.compaction.minor.idle", "5m", PropertyType.TIMEDURATION,
"After a tablet has been idle (no mutations) for this time period it may have its "
+ "in-memory map flushed to disk in a minor compaction. There is no guarantee an idle "
- + "tablet will be compacted."),
- TABLE_MINC_MAX_MERGE_FILE_SIZE("table.compaction.minor.merge.file.size.max", "1",
- PropertyType.MEMORY,
- "The max file size used for a merging minor compaction. The value of 0 is no max "
- + "file size. The default value is 1 byte, which should disable merging minor compactions."),
- TABLE_SCAN_MAXMEM("table.scan.max.memory", "512K", PropertyType.MEMORY,
+ + "tablet will be compacted.",
+ "1.3.5"),
+ TABLE_COMPACTION_DISPATCHER("table.compaction.dispatcher",
+ SimpleCompactionDispatcher.class.getName(), PropertyType.CLASSNAME,
+ "A configurable dispatcher that decides what compaction service a table should use.",
+ "2.1.0"),
+ TABLE_COMPACTION_DISPATCHER_OPTS("table.compaction.dispatcher.opts.", null, PropertyType.PREFIX,
+ "Options for the table compaction dispatcher", "2.1.0"),
+ TABLE_COMPACTION_SELECTION_EXPIRATION("table.compaction.selection.expiration.ms", "2m",
+ PropertyType.TIMEDURATION,
+ "User compactions select files and are then queued for compaction, preventing these files "
+ + "from being used in system compactions. This timeout allows system compactions to cancel "
+ + "the hold queued user compactions have on files, when its queued for more than the "
+ + "specified time. If a system compaction cancels a hold and runs, then the user compaction"
+ + " can reselect and hold files after the system compaction runs.",
+ "2.1.0"),
+ TABLE_COMPACTION_SELECTOR("table.compaction.selector", "", PropertyType.CLASSNAME,
+ "A configurable selector for a table that can periodically select file for mandatory "
+ + "compaction, even if the files do not meet the compaction ratio.",
+ "2.1.0"),
+ TABLE_COMPACTION_SELECTOR_OPTS("table.compaction.selector.opts.", null, PropertyType.PREFIX,
+ "Options for the table compaction dispatcher", "2.1.0"),
+ TABLE_COMPACTION_CONFIGURER("table.compaction.configurer", "", PropertyType.CLASSNAME,
+ "A plugin that can dynamically configure compaction output files based on input files.",
+ "2.1.0"),
+ TABLE_COMPACTION_CONFIGURER_OPTS("table.compaction.configurer.opts.", null, PropertyType.PREFIX,
+ "Options for the table compaction configuror", "2.1.0"),
+ @Deprecated(since = "2.1.0", forRemoval = true)
+ @ReplacedBy(property = TABLE_COMPACTION_SELECTOR)
+ TABLE_COMPACTION_STRATEGY("table.majc.compaction.strategy",
+ "org.apache.accumulo.tserver.compaction.DefaultCompactionStrategy", PropertyType.CLASSNAME,
+ "See {% jlink -f org.apache.accumulo.core.spi.compaction}", "1.6.0"),
+ @Deprecated(since = "2.1.0", forRemoval = true)
+ @ReplacedBy(property = TABLE_COMPACTION_SELECTOR_OPTS)
+ TABLE_COMPACTION_STRATEGY_PREFIX("table.majc.compaction.strategy.opts.", null,
+ PropertyType.PREFIX,
+ "Properties in this category are used to configure the compaction strategy.", "1.6.0"),
+ // Crypto-related properties
+ @Experimental
+ TABLE_CRYPTO_PREFIX("table.crypto.opts.", null, PropertyType.PREFIX,
+ "Properties related to on-disk file encryption.", "2.1.0"),
+ @Experimental
+ @Sensitive
+ TABLE_CRYPTO_SENSITIVE_PREFIX("table.crypto.opts.sensitive.", null, PropertyType.PREFIX,
+ "Sensitive properties related to on-disk file encryption.", "2.1.0"),
+ TABLE_SCAN_DISPATCHER("table.scan.dispatcher", SimpleScanDispatcher.class.getName(),
+ PropertyType.CLASSNAME,
+ "This class is used to dynamically dispatch scans to configured scan executors. Configured "
+ + "classes must implement {% jlink " + ScanDispatcher.class.getName() + " %} See "
+ + "[scan executors]({% durl administration/scan-executors %}) for an overview of why"
+ + " and how to use this property. This property is ignored for the root and metadata"
+ + " table. The metadata table always dispatches to a scan executor named `meta`.",
+ "2.0.0"),
+ TABLE_SCAN_DISPATCHER_OPTS("table.scan.dispatcher.opts.", null, PropertyType.PREFIX,
+ "Options for the table scan dispatcher", "2.0.0"),
+ TABLE_SCAN_MAXMEM("table.scan.max.memory", "512k", PropertyType.BYTES,
"The maximum amount of memory that will be used to cache results of a client query/scan. "
- + "Once this limit is reached, the buffered data is sent to the client."),
+ + "Once this limit is reached, the buffered data is sent to the client.",
+ "1.3.5"),
- TABLE_FILE_TYPE("table.file.type", RFile.EXTENSION, PropertyType.STRING,
+ TABLE_FILE_TYPE("table.file.type", RFile.EXTENSION, PropertyType.FILENAME_EXT,
- "Change the type of file a table writes"),
- TABLE_LOAD_BALANCER("table.balancer",
- "org.apache.accumulo.server.master.balancer.DefaultLoadBalancer", PropertyType.STRING,
+ "Change the type of file a table writes", "1.3.5"),
+ TABLE_LOAD_BALANCER("table.balancer", "org.apache.accumulo.core.spi.balancer.SimpleLoadBalancer",
+ PropertyType.STRING,
"This property can be set to allow the LoadBalanceByTable load balancer"
- + " to change the called Load Balancer for this table"),
+ + " to change the called Load Balancer for this table",
+ "1.3.5"),
TABLE_FILE_COMPRESSION_TYPE("table.file.compress.type", "gz", PropertyType.STRING,
- "One of gz,snappy,lzo,none"),
- TABLE_FILE_COMPRESSED_BLOCK_SIZE("table.file.compress.blocksize", "100K", PropertyType.MEMORY,
- "Similar to the hadoop io.seqfile.compress.blocksize setting, so that"
- + " files have better query performance. The maximum value for this is "
- + Integer.MAX_VALUE + ". (This setting is the size threshold prior to"
- + " compression, and applies even compression is disabled.)"),
- TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX("table.file.compress.blocksize.index", "128K",
- PropertyType.MEMORY,
- "Determines how large index blocks can be in files that support"
- + " multilevel indexes. The maximum value for this is " + Integer.MAX_VALUE
- + ". (This setting is the size threshold prior to compression, and applies"
- + " even compression is disabled.)"),
- TABLE_FILE_BLOCK_SIZE("table.file.blocksize", "0B", PropertyType.MEMORY,
- "Overrides the hadoop dfs.block.size setting so that files have better"
- + " query performance. The maximum value for this is " + Integer.MAX_VALUE),
+ "Compression algorithm used on index and data blocks before they are"
+ + " written. Possible values: zstd, gz, snappy, bzip2, lzo, lz4, none",
+ "1.3.5"),
+ TABLE_FILE_COMPRESSED_BLOCK_SIZE("table.file.compress.blocksize", "100k", PropertyType.BYTES,
+ "The maximum size of data blocks in RFiles before they are compressed and written.", "1.3.5"),
+ TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX("table.file.compress.blocksize.index", "128k",
+ PropertyType.BYTES,
+ "The maximum size of index blocks in RFiles before they are compressed and written.",
+ "1.4.0"),
+ TABLE_FILE_BLOCK_SIZE("table.file.blocksize", "0B", PropertyType.BYTES,
+ "The HDFS block size used when writing RFiles. When set to 0B, the"
+ + " value/defaults of HDFS property 'dfs.block.size' will be used.",
+ "1.3.5"),
TABLE_FILE_REPLICATION("table.file.replication", "0", PropertyType.COUNT,
- "Determines how many replicas to keep of a tables' files in HDFS. "
- + "When this value is LTE 0, HDFS defaults are used."),
+ "The number of replicas for a table's RFiles in HDFS. When set to 0, HDFS"
+ + " defaults are used.",
+ "1.3.5"),
TABLE_FILE_MAX("table.file.max", "15", PropertyType.COUNT,
- "Determines the max # of files each tablet in a table can have. When"
+ "The maximum number of RFiles each tablet in a table can have. When"
+ " adjusting this property you may want to consider adjusting"
+ " table.compaction.major.ratio also. Setting this property to 0 will make"
+ " it default to tserver.scan.files.open.max-1, this will prevent a tablet"
diff --cc core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java
index b13a026c8c,f8c468d87c..5d52e58965
--- a/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java
@@@ -26,10 -24,12 +26,11 @@@ import java.util.function.Function
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.IntStream;
import java.util.stream.Stream;
-import org.apache.accumulo.core.Constants;
+ import org.apache.accumulo.core.file.rfile.RFile;
-import org.apache.accumulo.core.util.Pair;
-import org.apache.commons.lang.math.IntRange;
+import org.apache.commons.lang3.Range;
import org.apache.hadoop.fs.Path;
import com.google.common.base.Preconditions;
diff --cc core/src/main/java/org/apache/accumulo/core/iteratorsImpl/system/MapFileIterator.java
index 3fddf87350,0000000000..bc3d317903
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/system/MapFileIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/iteratorsImpl/system/MapFileIterator.java
@@@ -1,174 -1,0 +1,121 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.iteratorsImpl.system;
+
+import java.io.DataInputStream;
- import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.FileSKVIterator;
- import org.apache.accumulo.core.file.NoSuchMetaStoreException;
+import org.apache.accumulo.core.file.blockfile.impl.CacheProvider;
- import org.apache.accumulo.core.file.map.MapFileUtil;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
- import org.apache.hadoop.fs.Path;
- import org.apache.hadoop.io.MapFile.Reader;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
+
+public class MapFileIterator implements FileSKVIterator {
- private static final Logger log = LoggerFactory.getLogger(MapFileIterator.class);
+
- private Reader reader;
- private Value topValue;
- private Key topKey;
- private AtomicBoolean interruptFlag;
- private int interruptCheckCount = 0;
- private FileSystem fs;
- private String dirName;
++ private static final String MSG = "Map files are not supported";
+
- public MapFileIterator(FileSystem fs, String dir, Configuration conf) throws IOException {
- this.reader = MapFileUtil.openMapFile(fs, dir, conf);
- this.fs = fs;
- this.dirName = dir;
++ public MapFileIterator(FileSystem fs, String dir, Configuration conf) {
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
+ public void setInterruptFlag(AtomicBoolean flag) {
- this.interruptFlag = flag;
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
+ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options,
- IteratorEnvironment env) throws IOException {
- throw new UnsupportedOperationException();
++ IteratorEnvironment env) {
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
+ public boolean hasTop() {
- return topKey != null;
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
- public void next() throws IOException {
- if (interruptFlag != null && interruptCheckCount++ % 100 == 0 && interruptFlag.get()) {
- throw new IterationInterruptedException();
- }
-
- reader.next(topKey, topValue);
++ public void next() {
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
- public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive)
- throws IOException {
- if (!columnFamilies.isEmpty() || inclusive) {
- throw new IllegalArgumentException("I do not know how to filter column families");
- }
-
- if (range == null) {
- throw new IllegalArgumentException("Cannot seek to null range");
- }
-
- if (interruptFlag != null && interruptFlag.get()) {
- throw new IterationInterruptedException();
- }
-
- Key key = range.getStartKey();
- if (key == null) {
- key = new Key();
- }
-
- reader.seek(key);
-
- while (hasTop() && range.beforeStartKey(getTopKey())) {
- next();
- }
++ public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) {
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
+ public Key getTopKey() {
- return topKey;
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
+ public Value getTopValue() {
- return topValue;
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
+ public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
- try {
- @SuppressWarnings("deprecation")
- SortedKeyValueIterator<Key,Value> other = env.reserveMapFileReader(dirName);
- ((InterruptibleIterator) other).setInterruptFlag(interruptFlag);
- log.debug("deep copying MapFile: {} -> {}", this, other);
- return other;
- } catch (IOException e) {
- log.error("failed to clone map file reader", e);
- throw new RuntimeException(e);
- }
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
- public Key getFirstKey() throws IOException {
- throw new UnsupportedOperationException();
++ public Key getFirstKey() {
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
- public Key getLastKey() throws IOException {
- throw new UnsupportedOperationException();
++ public Key getLastKey() {
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
- public DataInputStream getMetaStore(String name) throws IOException {
- Path path = new Path(this.dirName, name);
- if (!fs.exists(path)) {
- throw new NoSuchMetaStoreException("name = " + name);
- }
- return fs.open(path);
++ public DataInputStream getMetaStore(String name) {
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
- public void closeDeepCopies() throws IOException {
- // nothing to do, deep copies are externally managed/closed
++ public void closeDeepCopies() {
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
- public void close() throws IOException {
- reader.close();
++ public void close() {
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
+ public FileSKVIterator getSample(SamplerConfigurationImpl sampleConfig) {
- return null;
++ throw new UnsupportedOperationException(MSG);
+ }
+
+ @Override
- public void setCacheProvider(CacheProvider cacheProvider) {}
++ public void setCacheProvider(CacheProvider cacheProvider) {
++ throw new UnsupportedOperationException(MSG);
++ }
+}
diff --cc core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java
index 514fd1e95c,53ef7c4003..41a651ebe6
--- a/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java
@@@ -27,9 -25,11 +27,10 @@@ import java.util.Set
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import org.apache.accumulo.core.WithTestNames;
+ import org.apache.accumulo.core.file.rfile.RFile;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestName;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
import com.google.common.base.Joiner;