You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by dl...@apache.org on 2025/12/05 16:42:15 UTC
(accumulo) branch main updated (1c163b5505 -> a5cf1e622d)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
from 1c163b5505 Merge branch '2.1'
add 88abe226fd Log warning when durability not sync for root and meta tables (#5988)
new a5cf1e622d Merge branch '2.1'
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/accumulo/core/conf/Property.java | 17 +++++++--
.../apache/accumulo/core/file/FileOperations.java | 43 ++++++++++++++++------
.../accumulo/core/file/rfile/RFileOperations.java | 16 ++++++++
.../accumulo/server/compaction/FileCompactor.java | 7 ++--
.../server/init/FileSystemInitializer.java | 8 ++--
.../accumulo/tserver/TabletClientHandler.java | 11 +++++-
6 files changed, 78 insertions(+), 24 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
Posted by dl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit a5cf1e622d8eb587fca71288507c7bd24956ad7e
Merge: 1c163b5505 88abe226fd
Author: Dave Marion <dl...@apache.org>
AuthorDate: Fri Dec 5 16:42:02 2025 +0000
Merge branch '2.1'
.../org/apache/accumulo/core/conf/Property.java | 17 +++++++--
.../apache/accumulo/core/file/FileOperations.java | 43 ++++++++++++++++------
.../accumulo/core/file/rfile/RFileOperations.java | 16 ++++++++
.../accumulo/server/compaction/FileCompactor.java | 7 ++--
.../server/init/FileSystemInitializer.java | 8 ++--
.../accumulo/tserver/TabletClientHandler.java | 11 +++++-
6 files changed, 78 insertions(+), 24 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 8c9a3b1ac2,9cd510fbd0..141b02b273
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -1122,68 -1256,77 +1122,77 @@@ public enum Property
TABLE_BLOOM_ERRORRATE("table.bloom.error.rate", "0.5%", PropertyType.FRACTION,
"Bloom filter error rate.", "1.3.5"),
TABLE_BLOOM_KEY_FUNCTOR("table.bloom.key.functor",
- "org.apache.accumulo.core.file.keyfunctor.RowFunctor", PropertyType.CLASSNAME,
- "A function that can transform the key prior to insertion and check of"
- + " bloom filter. org.apache.accumulo.core.file.keyfunctor.RowFunctor,"
- + " org.apache.accumulo.core.file.keyfunctor.ColumnFamilyFunctor, and"
- + " org.apache.accumulo.core.file.keyfunctor.ColumnQualifierFunctor are"
- + " allowable values. One can extend any of the above mentioned classes to"
- + " perform specialized parsing of the key.",
- "1.3.5"),
+ "org.apache.accumulo.core.file.keyfunctor.RowFunctor", PropertyType.CLASSNAME, """
+ A function that can transform the key prior to insertion and check of \
+ bloom filter. org.apache.accumulo.core.file.keyfunctor.RowFunctor, \
+ org.apache.accumulo.core.file.keyfunctor.ColumnFamilyFunctor, and \
+ org.apache.accumulo.core.file.keyfunctor.ColumnQualifierFunctor are \
+ allowable values. One can extend any of the above mentioned classes to \
+ perform specialized parsing of the key.
+ """, "1.3.5"),
TABLE_BLOOM_HASHTYPE("table.bloom.hash.type", "murmur", PropertyType.STRING,
"The bloom filter hash type.", "1.3.5"),
- TABLE_BULK_MAX_TABLETS("table.bulk.max.tablets", "0", PropertyType.COUNT,
- "The maximum number of tablets allowed for one bulk import file. Value of 0 is Unlimited. "
- + "This property is only enforced in the new bulk import API.",
- "2.1.0"),
TABLE_BULK_SKIP_THRESHOLD("table.bulk.metadata.skip.distance", "0", PropertyType.COUNT,
- "When performing bulk v2 imports to a table, the Manager iterates over the tables metadata"
- + " tablets sequentially. When importing files into a small table or into all or a majority"
- + " of tablets of a large table then the tablet metadata information for most tablets will be needed."
- + " However, when importing files into a small number of non-contiguous tablets in a large table, then"
- + " the Manager will look at each tablets metadata when it could be skipped. The value of this"
- + " property tells the Manager if, and when, it should set up a new scanner over the metadata"
- + " table instead of just iterating over tablet metadata to find the matching tablet. Setting up"
- + " a new scanner is analogous to performing a seek in an iterator, but it has a cost. A value of zero (default) disables"
- + " this feature. A non-zero value enables this feature and the Manager will setup a new scanner"
- + " when the tablet metadata distance is above the supplied value.",
+ """
+ When performing bulk v2 imports to a table, the Manager iterates over the tables metadata \
+ tablets sequentially. When importing files into a small table or into all or a majority \
+ of tablets of a large table then the tablet metadata information for most tablets will be needed. \
+ However, when importing files into a small number of non-contiguous tablets in a large table, then \
+ the Manager will look at each tablets metadata when it could be skipped. The value of this \
+ property tells the Manager if, and when, it should set up a new scanner over the metadata \
+ table instead of just iterating over tablet metadata to find the matching tablet. Setting up \
+ a new scanner is analogous to performing a seek in an iterator, but it has a cost. A value of zero (default) disables \
+ this feature. A non-zero value enables this feature and the Manager will setup a new scanner \
+ when the tablet metadata distance is above the supplied value.
+ """,
"2.1.4"),
- TABLE_DURABILITY("table.durability", "sync", PropertyType.DURABILITY,
- "The durability of writes to tables includes ensuring that mutations written"
- + " by clients are persisted in the write-ahead log and that files written"
- + " during a compaction are persisted to disk successfully. This property only"
- + " configures the durability used to write to the write-ahead log. Legal"
- + " values are: none, which skips the write-ahead log; log, which sends the"
- + " data to the write-ahead log, but does nothing to make it durable; flush,"
- + " which pushes data out of the JVM (likely to page cache); and sync, which"
- + " ensures that each mutation is written to the physical disk. To configure"
- + " the durability of files written during minor and major compactions, set the"
- + " Hadoop property \"dfs.datanode.synconclose\" to \"true\". This will ensure"
- + " that the blocks of the files in HDFS are written to the physical disk as"
- + " the compaction output files are written (Note that this may only apply"
- + " to replicated files in HDFS).",
- "1.7.0"),
+ TABLE_DURABILITY("table.durability", "sync", PropertyType.DURABILITY, """
- The durability used to write to the write-ahead log. Legal values are: \
- none, which skips the write-ahead log; log, which sends the data to the \
- write-ahead log, but does nothing to make it durable; flush, which pushes \
- data to the file system; and sync, which ensures the data is written to disk. \
++ The durability of writes to tables includes ensuring that mutations written \
++ by clients are persisted in the write-ahead log and that files written \
++ during a compaction are persisted to disk successfully. This property only \
++ configures the durability used to write to the write-ahead log. Legal \
++ values are: none, which skips the write-ahead log; log, which sends the \
++ data to the write-ahead log, but does nothing to make it durable; flush, \
++ which pushes data out of the JVM (likely to page cache); and sync, which \
++ ensures that each mutation is written to the physical disk. To configure \
++ the durability of files written during minor and major compactions, set the \
++ Hadoop property \"dfs.datanode.synconclose\" to \"true\". This will ensure \
++ that the blocks of the files in HDFS are written to the physical disk as \
++ the compaction output files are written (Note that this may only apply \
++ to replicated files in HDFS). \
+ """, "1.7.0"),
- TABLE_FAILURES_IGNORE("table.failures.ignore", "false", PropertyType.BOOLEAN,
- "If you want queries for your table to hang or fail when data is missing"
- + " from the system, then set this to false. When this set to true missing"
- + " data will be reported but queries will still run possibly returning a"
- + " subset of the data.",
- "1.3.5"),
+ TABLE_FAILURES_IGNORE("table.failures.ignore", "false", PropertyType.BOOLEAN, """
+ If you want queries for your table to hang or fail when data is missing \
+ from the system, then set this to false. When this set to true missing \
+ data will be reported but queries will still run possibly returning a \
+ subset of the data. \
+ """, "1.3.5"),
TABLE_DEFAULT_SCANTIME_VISIBILITY("table.security.scan.visibility.default", "",
- PropertyType.STRING,
- "The security label that will be assumed at scan time if an entry does"
- + " not have a visibility expression.\n"
- + "Note: An empty security label is displayed as []. The scan results"
- + " will show an empty visibility even if the visibility from this"
- + " setting is applied to the entry.\n"
- + "CAUTION: If a particular key has an empty security label AND its"
- + " table's default visibility is also empty, access will ALWAYS be"
- + " granted for users with permission to that table. Additionally, if this"
- + " field is changed, all existing data with an empty visibility label"
- + " will be interpreted with the new label on the next scan.",
- "1.3.5"),
+ PropertyType.STRING, """
+ The security label that will be assumed at scan time if an entry does \
+ not have a visibility expression.
+ Note: An empty security label is displayed as []. The scan results \
+ will show an empty visibility even if the visibility from this \
+ setting is applied to the entry.
+ CAUTION: If a particular key has an empty security label AND its \
+ table's default visibility is also empty, access will ALWAYS be \
+ granted for users with permission to that table. Additionally, if this \
+ field is changed, all existing data with an empty visibility label \
+ will be interpreted with the new label on the next scan. \
+ """, "1.3.5"),
TABLE_LOCALITY_GROUPS("table.groups.enabled", "", PropertyType.STRING,
"A comma separated list of locality group names to enable for this table.", "1.3.5"),
- TABLE_CONSTRAINT_PREFIX("table.constraint.", null, PropertyType.PREFIX,
- "Properties in this category are per-table properties that add"
- + " constraints to a table. These properties start with the category"
- + " prefix, followed by a number, and their values correspond to a fully"
- + " qualified Java class that implements the Constraint interface.\nFor example:\n"
- + "table.constraint.1 = org.apache.accumulo.core.constraints.MyCustomConstraint\n"
- + "and:\n table.constraint.2 = my.package.constraints.MySecondConstraint.",
- "1.3.5"),
+ TABLE_CONSTRAINT_PREFIX("table.constraint.", null, PropertyType.PREFIX, """
+ Properties in this category are per-table properties that add \
+ constraints to a table. These properties start with the category \
+ prefix, followed by a number, and their values correspond to a fully \
+ qualified Java class that implements the Constraint interface.
+ For example:
+ table.constraint.1 = org.apache.accumulo.core.constraints.MyCustomConstraint
+ and:
+ table.constraint.2 = my.package.constraints.MySecondConstraint.
+ """, "1.3.5"),
TABLE_INDEXCACHE_ENABLED("table.cache.index.enable", "true", PropertyType.BOOLEAN,
"Determines whether index block cache is enabled for a table.", "1.3.5"),
TABLE_BLOCKCACHE_ENABLED("table.cache.block.enable", "false", PropertyType.BOOLEAN,
diff --cc core/src/main/java/org/apache/accumulo/core/file/FileOperations.java
index 309b7b3cab,ce5c392fdc..ecbc595664
--- a/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/FileOperations.java
@@@ -29,11 -29,11 +29,12 @@@ import org.apache.accumulo.core.conf.Ac
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Range;
+ import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.file.blockfile.impl.CacheProvider;
import org.apache.accumulo.core.file.rfile.RFile;
+import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.metadata.UnreferencedTabletFile;
import org.apache.accumulo.core.spi.crypto.CryptoService;
-import org.apache.accumulo.core.util.ratelimit.RateLimiter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@@ -169,13 -169,15 +170,14 @@@ public abstract class FileOperations
return new ReaderBuilder();
}
- public static class FileOptions {
+ protected static class FileOptions {
// objects used by all
public final AccumuloConfiguration tableConfiguration;
- public final String filename;
+ public final TabletFile file;
public final FileSystem fs;
public final Configuration fsConf;
- public final RateLimiter rateLimiter;
// writer only objects
+ private final TableId tableId;
public final String compression;
public final FSDataOutputStream outputStream;
public final boolean enableAccumuloStart;
@@@ -190,15 -192,17 +192,16 @@@
public final boolean inclusive;
public final boolean dropCacheBehind;
- protected FileOptions(AccumuloConfiguration tableConfiguration, TabletFile file, FileSystem fs,
- Configuration fsConf, String compression, FSDataOutputStream outputStream,
- boolean enableAccumuloStart, CacheProvider cacheProvider, Cache<String,Long> fileLenCache,
- boolean seekToBeginning, CryptoService cryptoService, Range range,
- Set<ByteSequence> columnFamilies, boolean inclusive, boolean dropCacheBehind) {
- public FileOptions(TableId tableId, AccumuloConfiguration tableConfiguration, String filename,
- FileSystem fs, Configuration fsConf, RateLimiter rateLimiter, String compression,
++ protected FileOptions(TableId tableId, AccumuloConfiguration tableConfiguration,
++ TabletFile file, FileSystem fs, Configuration fsConf, String compression,
+ FSDataOutputStream outputStream, boolean enableAccumuloStart, CacheProvider cacheProvider,
+ Cache<String,Long> fileLenCache, boolean seekToBeginning, CryptoService cryptoService,
+ Range range, Set<ByteSequence> columnFamilies, boolean inclusive, boolean dropCacheBehind) {
+ this.tableId = tableId;
this.tableConfiguration = tableConfiguration;
- this.filename = filename;
+ this.file = Objects.requireNonNull(file);
this.fs = fs;
this.fsConf = fsConf;
- this.rateLimiter = rateLimiter;
this.compression = compression;
this.outputStream = outputStream;
this.enableAccumuloStart = enableAccumuloStart;
@@@ -273,10 -285,12 +280,11 @@@
* Helper class extended by both writers and readers.
*/
public static class FileHelper {
+ private TableId tableId;
private AccumuloConfiguration tableConfiguration;
- private String filename;
+ private TabletFile file;
private FileSystem fs;
private Configuration fsConf;
- private RateLimiter rateLimiter;
private CryptoService cryptoService;
private boolean dropCacheBehind = false;
@@@ -312,27 -336,29 +325,28 @@@
protected FileOptions toWriterBuilderOptions(String compression,
FSDataOutputStream outputStream, boolean startEnabled) {
- return new FileOptions(tableConfiguration, file, fs, fsConf, compression, outputStream,
- startEnabled, NULL_PROVIDER, null, false, cryptoService, null, null, true,
- return new FileOptions(tableId, tableConfiguration, filename, fs, fsConf, rateLimiter,
- compression, outputStream, startEnabled, NULL_PROVIDER, null, false, cryptoService, null,
- null, true, dropCacheBehind);
++ return new FileOptions(tableId, tableConfiguration, file, fs, fsConf, compression,
++ outputStream, startEnabled, NULL_PROVIDER, null, false, cryptoService, null, null, true,
+ dropCacheBehind);
}
protected FileOptions toReaderBuilderOptions(CacheProvider cacheProvider,
Cache<String,Long> fileLenCache, boolean seekToBeginning) {
- return new FileOptions(tableConfiguration, file, fs, fsConf, null, null, false,
- return new FileOptions(tableId, tableConfiguration, filename, fs, fsConf, rateLimiter, null,
- null, false, cacheProvider == null ? NULL_PROVIDER : cacheProvider, fileLenCache,
- seekToBeginning, cryptoService, null, null, true, dropCacheBehind);
++ return new FileOptions(tableId, tableConfiguration, file, fs, fsConf, null, null, false,
+ cacheProvider == null ? NULL_PROVIDER : cacheProvider, fileLenCache, seekToBeginning,
+ cryptoService, null, null, true, dropCacheBehind);
}
protected FileOptions toIndexReaderBuilderOptions(Cache<String,Long> fileLenCache) {
- return new FileOptions(tableConfiguration, file, fs, fsConf, null, null, false, NULL_PROVIDER,
- fileLenCache, false, cryptoService, null, null, true, dropCacheBehind);
- return new FileOptions(tableId, tableConfiguration, filename, fs, fsConf, rateLimiter, null,
- null, false, NULL_PROVIDER, fileLenCache, false, cryptoService, null, null, true,
- dropCacheBehind);
++ return new FileOptions(tableId, tableConfiguration, file, fs, fsConf, null, null, false,
++ NULL_PROVIDER, fileLenCache, false, cryptoService, null, null, true, dropCacheBehind);
}
protected FileOptions toScanReaderBuilderOptions(Range range, Set<ByteSequence> columnFamilies,
boolean inclusive) {
- return new FileOptions(tableConfiguration, file, fs, fsConf, null, null, false, NULL_PROVIDER,
- null, false, cryptoService, range, columnFamilies, inclusive, dropCacheBehind);
- return new FileOptions(tableId, tableConfiguration, filename, fs, fsConf, rateLimiter, null,
- null, false, NULL_PROVIDER, null, false, cryptoService, range, columnFamilies, inclusive,
++ return new FileOptions(tableId, tableConfiguration, file, fs, fsConf, null, null, false,
++ NULL_PROVIDER, null, false, cryptoService, range, columnFamilies, inclusive,
+ dropCacheBehind);
}
protected AccumuloConfiguration getTableConfiguration() {
diff --cc core/src/main/java/org/apache/accumulo/core/file/rfile/RFileOperations.java
index 884a4f095a,c87b071642..0927b6b374
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFileOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFileOperations.java
@@@ -33,15 -35,17 +35,17 @@@ import org.apache.accumulo.core.file.Fi
import org.apache.accumulo.core.file.FileSKVIterator;
import org.apache.accumulo.core.file.FileSKVWriter;
import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile.CachableBuilder;
+import org.apache.accumulo.core.file.rfile.RFile.RFileSKVIterator;
import org.apache.accumulo.core.file.rfile.bcfile.BCFile;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.RootTable;
++import org.apache.accumulo.core.metadata.SystemTables;
+import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
import org.apache.accumulo.core.sample.impl.SamplerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
+ import org.apache.hadoop.fs.StreamCapabilities;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.slf4j.Logger;
@@@ -54,13 -58,15 +58,14 @@@ public class RFileOperations extends Fi
private static final Logger LOG = LoggerFactory.getLogger(RFileOperations.class);
private static final Collection<ByteSequence> EMPTY_CF_SET = Collections.emptySet();
+ private static final AtomicBoolean SYNC_CAPABILITY_LOGGED = new AtomicBoolean(false);
- private static RFile.Reader getReader(FileOptions options) throws IOException {
+ private static RFileSKVIterator getReader(FileOptions options) throws IOException {
CachableBuilder cb = new CachableBuilder()
- .fsPath(options.getFileSystem(), new Path(options.getFilename()), options.dropCacheBehind)
+ .fsPath(options.getFileSystem(), options.getFile().getPath(), options.dropCacheBehind)
.conf(options.getConfiguration()).fileLen(options.getFileLenCache())
- .cacheProvider(options.cacheProvider).readLimiter(options.getRateLimiter())
- .cryptoService(options.getCryptoService());
- return new RFile.Reader(cb);
+ .cacheProvider(options.cacheProvider).cryptoService(options.getCryptoService());
+ return RFile.getReader(cb, options.getFile());
}
@Override
@@@ -194,9 -200,21 +199,20 @@@
e.getMessage());
}
}
+
+ TableId tid = options.getTableId();
- if (tid != null && !SYNC_CAPABILITY_LOGGED.get()
- && (RootTable.ID.equals(tid) || MetadataTable.ID.equals(tid))) {
++ if (tid != null && !SYNC_CAPABILITY_LOGGED.get() && (SystemTables.ROOT.tableId().equals(tid)
++ || SystemTables.METADATA.tableId().equals(tid))) {
+ if (!outputStream.hasCapability(StreamCapabilities.HSYNC)) {
+ SYNC_CAPABILITY_LOGGED.set(true);
+ LOG.warn("File created for table {} does not support hsync. If dfs.datanode.synconclose"
+ + " is configured, then it may not work. dfs.datanode.synconclose is recommended for the"
+ + " root and metadata tables.", tid);
+ }
+ }
}
- BCFile.Writer _cbw = new BCFile.Writer(outputStream, options.getRateLimiter(), compression,
- conf, options.cryptoService);
+ BCFile.Writer _cbw = new BCFile.Writer(outputStream, compression, conf, options.cryptoService);
return new RFile.Writer(_cbw, (int) blockSize, (int) indexBlockSize, samplerConfig, sampler);
}
diff --cc server/base/src/main/java/org/apache/accumulo/server/compaction/FileCompactor.java
index 813744bee6,0d48f26799..9133fd1b0f
--- a/server/base/src/main/java/org/apache/accumulo/server/compaction/FileCompactor.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/compaction/FileCompactor.java
@@@ -355,15 -354,14 +355,16 @@@ public class FileCompactor implements C
final boolean isMinC = env.getIteratorScope() == IteratorUtil.IteratorScope.minc;
- final boolean dropCacheBehindOutput = !RootTable.ID.equals(this.extent.tableId())
- && !MetadataTable.ID.equals(this.extent.tableId())
- && ((isMinC && acuTableConf.getBoolean(Property.TABLE_MINC_OUTPUT_DROP_CACHE))
- || (!isMinC && acuTableConf.getBoolean(Property.TABLE_MAJC_OUTPUT_DROP_CACHE)));
+ final boolean dropCacheBehindOutput =
+ !SystemTables.ROOT.tableId().equals(this.extent.tableId())
+ && !SystemTables.METADATA.tableId().equals(this.extent.tableId())
+ && ((isMinC && acuTableConf.getBoolean(Property.TABLE_MINC_OUTPUT_DROP_CACHE))
+ || (!isMinC && acuTableConf.getBoolean(Property.TABLE_MAJC_OUTPUT_DROP_CACHE)));
- WriterBuilder outBuilder =
- fileFactory.newWriterBuilder().forFile(outputFile, ns, ns.getConf(), cryptoService)
- .withTableConfiguration(acuTableConf);
+ WriterBuilder outBuilder = fileFactory.newWriterBuilder().forTable(this.extent.tableId())
- .forFile(outputFile.getMetaInsert(), ns, ns.getConf(), cryptoService)
- .withTableConfiguration(acuTableConf).withRateLimiter(env.getWriteLimiter());
++ .forFile(outputFile, ns, ns.getConf(), cryptoService)
++ .withTableConfiguration(acuTableConf);
++
if (dropCacheBehindOutput) {
outBuilder.dropCachesBehind();
}
diff --cc server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java
index 0c5319553c,873ea84347..a0ac9726a2
--- a/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java
@@@ -155,29 -119,23 +155,29 @@@ public class FileSystemInitializer
// create table and default tablets directories
createDirectories(fs, rootTabletDirUri, tableMetadataTabletDirUri, defaultMetadataTabletDirUri,
- replicationTableDefaultTabletDirUri);
+ fateTableDefaultTabletDirUri, scanRefTableDefaultTabletDirUri);
- String ext = FileOperations.getNewFileExtension(DefaultConfiguration.getInstance());
+ // For a new system mark the fate tablet and scan ref tablet as always mergeable.
+ // Because this is a new system we can just use 0 for the time as that is what the Manager
+ // will initialize with when starting
+ var always = TabletMergeabilityMetadata.always(SteadyTime.from(Duration.ZERO));
+ InitialTablet fateTablet = createFateRefTablet(context, always);
+ InitialTablet scanRefTablet = createScanRefTablet(context, always);
- // populate the metadata tablet with info about the replication tablet
+ // populate the metadata tablet with info about the fate and scan ref tablets
+ String ext = FileOperations.getNewFileExtension(DefaultConfiguration.getInstance());
String metadataFileName = tableMetadataTabletDirUri + Path.SEPARATOR + "0_1." + ext;
- createMetadataFile(fs, metadataFileName, fateTablet, scanRefTablet);
- Tablet replicationTablet =
- new Tablet(REPL_TABLE_ID, replicationTableDefaultTabletDirName, null, null);
- createMetadataFile(fs, metadataFileName, siteConfig, REPL_TABLE_ID, replicationTablet);
++ createMetadataFile(fs, metadataFileName, scanRefTablet.tableId, fateTablet, scanRefTablet);
// populate the root tablet with info about the metadata table's two initial tablets
- Tablet tablesTablet = new Tablet(MetadataTable.ID, tableMetadataTabletDirName, null, splitPoint,
- metadataFileName);
- Tablet defaultTablet =
- new Tablet(MetadataTable.ID, defaultMetadataTabletDirName, splitPoint, null);
- createMetadataFile(fs, rootTabletFileUri, siteConfig, MetadataTable.ID, tablesTablet,
- defaultTablet);
+ // For the default tablet we want to make that mergeable, but don't make the TabletsSection
+ // tablet mergeable. This will prevent tablets from each either from being auto merged
+ InitialTablet tablesTablet = new InitialTablet(SystemTables.METADATA.tableId(),
+ TABLE_TABLETS_TABLET_DIR, null, SPLIT_POINT, TabletMergeabilityMetadata.never(),
+ StoredTabletFile.of(new Path(metadataFileName)).getMetadataPath());
+ InitialTablet defaultTablet = new InitialTablet(SystemTables.METADATA.tableId(),
+ defaultMetadataTabletDirName, SPLIT_POINT, null, always);
- createMetadataFile(fs, rootTabletFileUri, tablesTablet, defaultTablet);
++ createMetadataFile(fs, rootTabletFileUri, tablesTablet.tableId, tablesTablet, defaultTablet);
}
private void createDirectories(VolumeManager fs, String... dirs) throws IOException {
@@@ -218,24 -177,21 +218,24 @@@
}
}
-- private void createMetadataFile(VolumeManager volmanager, String fileName,
- AccumuloConfiguration conf, TableId tid, Tablet... tablets) throws IOException {
- // sort file contents in memory, then play back to the file
- TreeMap<Key,Value> sorted = new TreeMap<>();
- for (Tablet tablet : tablets) {
- createEntriesForTablet(sorted, tablet);
- }
- FileSystem fs = volmanager.getFileSystemByPath(new Path(fileName));
++ private void createMetadataFile(VolumeManager volmanager, String fileName, TableId tid,
+ InitialTablet... initialTablets) throws IOException {
+ AccumuloConfiguration conf = initConfig.getSiteConf();
+ ReferencedTabletFile file = ReferencedTabletFile.of(new Path(fileName));
+ FileSystem fs = volmanager.getFileSystemByPath(file.getPath());
CryptoService cs = CryptoFactoryLoader.getServiceForServer(conf);
- FileSKVWriter tabletWriter = FileOperations.getInstance().newWriterBuilder()
+ FileSKVWriter tabletWriter = FileOperations.getInstance().newWriterBuilder().forTable(tid)
- .forFile(fileName, fs, fs.getConf(), cs).withTableConfiguration(conf).build();
+ .forFile(file, fs, fs.getConf(), cs).withTableConfiguration(conf).build();
tabletWriter.startDefaultLocalityGroup();
+ TreeMap<Key,Value> sorted = new TreeMap<>();
+ for (InitialTablet initialTablet : initialTablets) {
+ // sort file contents in memory, then play back to the file
+ sorted.putAll(initialTablet.createEntries());
+ }
+
for (Map.Entry<Key,Value> entry : sorted.entrySet()) {
tabletWriter.append(entry.getKey(), entry.getValue());
}
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/TabletClientHandler.java
index 5de50cfe93,3644f6146c..97ea23793c
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletClientHandler.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletClientHandler.java
@@@ -195,6 -302,6 +195,7 @@@ public class TabletClientHandler implem
tableId = keyExtent.tableId();
if (sameTable || security.canWrite(us.getCredentials(), tableId,
server.getContext().getNamespaceId(tableId))) {
++ logDurabilityWarning(keyExtent, us.durability);
long t2 = System.currentTimeMillis();
us.authTimes.addStat(t2 - t1);
us.currentTablet = server.getOnlineTablet(keyExtent);
@@@ -320,9 -426,9 +321,10 @@@
Tablet tablet = entry.getKey();
Durability durability =
DurabilityImpl.resolveDurabilty(us.durability, tablet.getDurability());
- logDurabilityWarning(tablet, durability);
++ logDurabilityWarning(tablet.getExtent(), durability);
List<Mutation> mutations = entry.getValue();
if (!mutations.isEmpty()) {
+ preppedMutations += mutations.size();
try {
server.updateMetrics.addMutationArraySize(mutations.size());
@@@ -634,9 -815,8 +636,9 @@@
} else {
final Durability durability =
DurabilityImpl.resolveDurabilty(sess.durability, tablet.getDurability());
-
- logDurabilityWarning(tablet, durability);
++ logDurabilityWarning(tablet.getExtent(), durability);
List<Mutation> mutations = Collections.unmodifiableList(entry.getValue());
+ preppedMutions += mutations.size();
if (!mutations.isEmpty()) {
PreparedMutations prepared = tablet.prepareMutationsForCommit(
@@@ -1397,4 -1636,11 +1399,11 @@@
return handleTimeout(sessionId);
}
}
+
- private void logDurabilityWarning(Tablet tablet, Durability durability) {
- if (tablet.getExtent().isMeta() && durability != Durability.SYNC) {
++ private void logDurabilityWarning(KeyExtent tablet, Durability durability) {
++ if (tablet.isMeta() && durability != Durability.SYNC) {
+ log.warn("Property {} is not set to 'sync' for table {}", Property.TABLE_DURABILITY.getKey(),
- tablet.getExtent().tableId());
++ tablet.tableId());
+ }
+ }
}