You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by dl...@apache.org on 2026/07/07 15:57:11 UTC
(accumulo) branch main updated (6293e0cc1b -> 0466e7421b)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
from 6293e0cc1b Merge branch '2.1'
add de83b942f0 Set initial capacity on HashSet/HashMap when known (#6451)
new 0466e7421b Merge branch '2.1'
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../accumulo/core/client/MutationsRejectedException.java | 4 ++--
.../apache/accumulo/core/client/rfile/RFileWriter.java | 6 +++---
.../accumulo/core/clientImpl/TableOperationsImpl.java | 8 ++++----
.../java/org/apache/accumulo/core/data/Condition.java | 4 ++--
.../core/data/constraints/VisibilityConstraint.java | 2 +-
.../java/org/apache/accumulo/core/file/rfile/RFile.java | 2 +-
.../system/ColumnFamilySkippingIterator.java | 3 +--
.../accumulo/core/metadata/schema/TabletMetadata.java | 7 ++++---
.../org/apache/accumulo/core/util/LocalityGroupUtil.java | 2 +-
.../core/util/compaction/ExternalCompactionUtil.java | 15 ++++++++-------
server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java | 10 ++++++----
.../apache/accumulo/gc/GarbageCollectionAlgorithm.java | 2 +-
.../org/apache/accumulo/tserver/TabletClientHandler.java | 4 +++-
.../apache/accumulo/tserver/ThriftScanClientHandler.java | 4 ++--
.../java/org/apache/accumulo/tserver/tablet/Tablet.java | 7 ++++---
15 files changed, 43 insertions(+), 37 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
Posted by dl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 0466e7421bcdfc18b78c05cbde8678a840ee6d4d
Merge: 6293e0cc1b de83b942f0
Author: Dave Marion <dl...@apache.org>
AuthorDate: Tue Jul 7 15:57:01 2026 +0000
Merge branch '2.1'
.../accumulo/core/client/MutationsRejectedException.java | 4 ++--
.../apache/accumulo/core/client/rfile/RFileWriter.java | 6 +++---
.../accumulo/core/clientImpl/TableOperationsImpl.java | 8 ++++----
.../java/org/apache/accumulo/core/data/Condition.java | 4 ++--
.../core/data/constraints/VisibilityConstraint.java | 2 +-
.../java/org/apache/accumulo/core/file/rfile/RFile.java | 2 +-
.../system/ColumnFamilySkippingIterator.java | 3 +--
.../accumulo/core/metadata/schema/TabletMetadata.java | 7 ++++---
.../org/apache/accumulo/core/util/LocalityGroupUtil.java | 2 +-
.../core/util/compaction/ExternalCompactionUtil.java | 15 ++++++++-------
server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java | 10 ++++++----
.../apache/accumulo/gc/GarbageCollectionAlgorithm.java | 2 +-
.../org/apache/accumulo/tserver/TabletClientHandler.java | 4 +++-
.../apache/accumulo/tserver/ThriftScanClientHandler.java | 4 ++--
.../java/org/apache/accumulo/tserver/tablet/Tablet.java | 7 ++++---
15 files changed, 43 insertions(+), 37 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/data/constraints/VisibilityConstraint.java
index c9fcdc235b,e814782f2e..c4ec58dcda
--- a/core/src/main/java/org/apache/accumulo/core/data/constraints/VisibilityConstraint.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/constraints/VisibilityConstraint.java
@@@ -60,10 -64,10 +60,10 @@@ public class VisibilityConstraint imple
HashSet<String> ok = null;
if (updates.size() > 1) {
- ok = new HashSet<>();
+ ok = new HashSet<>(updates.size());
}
- VisibilityEvaluator ve = null;
+ BytesAccess.BytesEvaluator ve = null;
for (ColumnUpdate update : updates) {
diff --cc core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index 3ae83c4eca,dd11a95df8..4977d4cf35
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@@ -828,14 -535,15 +828,15 @@@ public class TabletMetadata
* pulled from org.apache.accumulo.server.manager.LiveTServerSet
*/
public static synchronized Set<TServerInstance> getLiveTServers(ClientContext context) {
- final Set<TServerInstance> liveServers = new HashSet<>();
- for (ServiceLockPath slp : context.getServerPaths().getTabletServer(ResourceGroupPredicate.ANY,
- AddressSelector.all(), true)) {
- final String path = context.getZooKeeperRoot() + Constants.ZTSERVERS;
- final List<String> children = context.getZooCache().getChildren(path);
- final Set<TServerInstance> liveServers = new HashSet<>(children.size());
++ Set<ServiceLockPath> locks = context.getServerPaths()
++ .getTabletServer(ResourceGroupPredicate.ANY, AddressSelector.all(), true);
++ final Set<TServerInstance> liveServers = new HashSet<>(locks.size());
++ for (ServiceLockPath slp : locks) {
- for (String child : children) {
- checkServer(context, path, child).ifPresent(liveServers::add);
+ checkTabletServer(context, slp).ifPresent(liveServers::add);
}
- log.trace("Found {} live tservers at ZK path: {}", liveServers.size(), path);
+ log.trace("Found {} live tservers in ZooKeeper", liveServers.size());
return liveServers;
}
diff --cc core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
index bffee1f518,c6be864ce8..7c8e878531
--- a/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java
@@@ -266,16 -257,18 +266,17 @@@ public class ExternalCompactionUtil
getCompactionIdsRunningOnCompactors(ClientContext context) {
final ExecutorService executor = ThreadPools.getServerThreadPools()
.getPoolBuilder(COMPACTOR_RUNNING_COMPACTION_IDS_POOL).numCoreThreads(16).build();
- List<Future<ExternalCompactionId>> futures = new ArrayList<>();
- context.getServerPaths().getCompactor(ResourceGroupPredicate.ANY, AddressSelector.all(), true)
- .forEach(slp -> {
- final HostAndPort hp = HostAndPort.fromString(slp.getServer());
- futures.add(executor.submit(() -> getRunningCompactionId(hp, context)));
- });
- var compactors = getCompactorAddrs(context);
++ Set<ServiceLockPath> compactors = context.getServerPaths()
++ .getCompactor(ResourceGroupPredicate.ANY, AddressSelector.all(), true);
+ List<Future<ExternalCompactionId>> futures = new ArrayList<>(compactors.size());
-
- compactors.forEach((q, hp) -> {
- hp.forEach(hostAndPort -> {
- futures.add(executor.submit(() -> getRunningCompactionId(hostAndPort, context)));
- });
++ compactors.forEach(slp -> {
++ final HostAndPort hp = HostAndPort.fromString(slp.getServer());
++ futures.add(executor.submit(() -> getRunningCompactionId(hp, context)));
+ });
executor.shutdown();
- HashSet<ExternalCompactionId> runningIds = new HashSet<>();
+ HashSet<ExternalCompactionId> runningIds = new HashSet<>(compactors.size() / 4);
futures.forEach(future -> {
try {
diff --cc server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
index dbef6f904b,cbd968245e..0fe140ce33
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
@@@ -233,12 -243,14 +233,13 @@@ public class GCRun implements GarbageCo
IllegalStateException ioe = null;
while (retries <= 10) {
try {
- zr.sync(tablesPath);
- var tables = zr.getChildren(tablesPath);
+ zr.sync(Constants.ZTABLES);
- final Map<TableId,TableState> tids = new HashMap<>();
- for (String table : zr.getChildren(Constants.ZTABLES)) {
++ var tables = zr.getChildren(Constants.ZTABLES);
+ final Map<TableId,TableState> tids = new HashMap<>(tables.size(), 1.0f);
+ for (String table : tables) {
TableId tableId = TableId.of(table);
TableState tableState = null;
- String statePath = context.getZooKeeperRoot() + Constants.ZTABLES + "/"
- + tableId.canonical() + Constants.ZTABLE_STATE;
+ String statePath = Constants.ZTABLES + "/" + tableId.canonical() + Constants.ZTABLE_STATE;
try {
byte[] state = zr.getData(statePath);
if (state == null) {
@@@ -552,12 -605,13 +553,13 @@@
@Override
public Set<TableId> getCandidateTableIDs() throws InterruptedException {
if (level == DataLevel.ROOT) {
- return Set.of(RootTable.ID);
+ return Set.of(SystemTables.ROOT.tableId());
} else if (level == DataLevel.METADATA) {
- return Set.of(MetadataTable.ID);
+ return Set.of(SystemTables.METADATA.tableId());
} else if (level == DataLevel.USER) {
- Set<TableId> tableIds = new HashSet<>();
- getTableIDs().forEach((k, v) -> {
+ var tids = getTableIDs();
+ Set<TableId> tableIds = new HashSet<>(tids.size(), 1.0f);
+ tids.forEach((k, v) -> {
if (v == TableState.ONLINE || v == TableState.OFFLINE) {
// Don't return tables that are NEW, DELETING, or in an
// UNKNOWN state.
diff --cc server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
index 7a8610a17b,1d5863ca9e..462d593cf4
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
@@@ -141,57 -145,55 +141,57 @@@ public class GarbageCollectionAlgorith
List<GcCandidate> candidateEntriesToBeDeleted = new ArrayList<>();
Set<TableId> tableIdsBefore = gce.getCandidateTableIDs();
- Set<TableId> tableIdsSeen = new HashSet<>();
+ Set<TableId> tableIdsSeen = new HashSet<>(tableIdsBefore.size());
- Iterator<Reference> iter = gce.getReferences().iterator();
- while (iter.hasNext()) {
- Reference ref = iter.next();
- tableIdsSeen.add(ref.getTableId());
+ try (Stream<Reference> references = gce.getReferences()) {
+ references.forEach(ref -> {
+ tableIdsSeen.add(ref.getTableId());
- if (ref.isDirectory()) {
- ServerColumnFamily.validateDirCol(ref.getMetadataEntry());
+ if (ref.isDirectory()) {
+ ServerColumnFamily.validateDirCol(ref.getMetadataPath());
- String dir = "/" + ref.getTableId() + "/" + ref.getMetadataEntry();
+ String dir = "/" + ref.getTableId() + "/" + ref.getMetadataPath();
- dir = makeRelative(dir, 2);
+ dir = makeRelative(dir, 2);
- GcCandidate gcTemp = candidateMap.remove(dir);
- if (gcTemp != null) {
- log.debug("Directory Candidate was still in use by dir ref: {}", dir);
- // Do not add dir candidates to candidateEntriesToBeDeleted as they are only created once.
- }
- } else {
- String reference = ref.getMetadataEntry();
- if (reference.startsWith("/")) {
- log.debug("Candidate {} has a relative path, prepend tableId {}", reference,
- ref.getTableId());
- reference = "/" + ref.getTableId() + ref.getMetadataEntry();
- } else if (!reference.contains(":") && !reference.startsWith("../")) {
- throw new RuntimeException("Bad file reference " + reference);
- }
+ GcCandidate gcTemp = candidateMap.remove(dir);
+ if (gcTemp != null) {
+ log.debug("Directory Candidate was still in use by dir ref: {}", dir);
+ // Do not add dir candidates to candidateEntriesToBeDeleted as they are only created
+ // once.
+ }
+ } else {
+ String reference = ref.getMetadataPath();
+ if (reference.startsWith("/")) {
+ log.debug("Candidate {} has a relative path, prepend tableId {}", reference,
+ ref.getTableId());
+ reference = "/" + ref.getTableId() + ref.getMetadataPath();
+ } else if (!reference.contains(":") && !reference.startsWith("../")) {
+ throw new RuntimeException("Bad file reference " + reference);
+ }
- String relativePath = makeRelative(reference, 3);
-
- // WARNING: This line is EXTREMELY IMPORTANT.
- // You MUST REMOVE candidates that are still in use
- GcCandidate gcTemp = candidateMap.remove(relativePath);
- if (gcTemp != null) {
- log.debug("File Candidate was still in use: {}", relativePath);
- // Prevent deletion of candidates that are still in use by scans, because they won't be
- // recreated once the scan is finished.
- if (!ref.isScan()) {
- candidateEntriesToBeDeleted.add(gcTemp);
+ String relativePath = makeRelative(reference, 3);
+
+ // WARNING: This line is EXTREMELY IMPORTANT.
+ // You MUST REMOVE candidates that are still in use
+ GcCandidate gcTemp = candidateMap.remove(relativePath);
+ if (gcTemp != null) {
+ log.debug("File Candidate was still in use: {}", relativePath);
+ // Prevent deletion of candidates that are still in use by scans, because they won't be
+ // recreated once the scan is finished.
+ if (!ref.isScan()) {
+ candidateEntriesToBeDeleted.add(gcTemp);
+ }
}
- }
- String dir = relativePath.substring(0, relativePath.lastIndexOf('/'));
- GcCandidate gcT = candidateMap.remove(dir);
- if (gcT != null) {
- log.debug("Directory Candidate was still in use by file ref: {}", relativePath);
- // Do not add dir candidates to candidateEntriesToBeDeleted as they are only created once.
+ String dir = relativePath.substring(0, relativePath.lastIndexOf('/'));
+ GcCandidate gcT = candidateMap.remove(dir);
+ if (gcT != null) {
+ log.debug("Directory Candidate was still in use by file ref: {}", relativePath);
+ // Do not add dir candidates to candidateEntriesToBeDeleted as they are only created
+ // once.
+ }
}
- }
+ });
}
Set<TableId> tableIdsAfter = gce.getCandidateTableIDs();
ensureAllTablesChecked(Collections.unmodifiableSet(tableIdsBefore),
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 64f7a4f302,63b3d95866..c1a4b5675a
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@@ -277,9 -341,9 +277,10 @@@ public class Tablet extends TabletBase
final AtomicLong maxTime = new AtomicLong(Long.MIN_VALUE);
final CommitSession commitSession = getTabletMemory().getCommitSession();
try {
- Set<String> absPaths = new HashSet<>();
- for (StoredTabletFile ref : metadata.getFiles()) {
- Set<String> absPaths = new HashSet<>(datafiles.size());
- for (TabletFile ref : datafiles.keySet()) {
- absPaths.add(ref.getPathStr());
++ Set<StoredTabletFile> files = metadata.getFiles();
++ Set<String> absPaths = new HashSet<>(files.size());
++ for (StoredTabletFile ref : files) {
+ absPaths.add(ref.getNormalizedPathStr());
}
tabletServer.recover(this.getTabletServer().getVolumeManager(), extent, logEntries,
@@@ -335,9 -395,10 +336,9 @@@
}
}
// make some closed references that represent the recovered logs
- currentLogs = new HashSet<>();
+ currentLogs = new HashSet<>(logEntries.size(), 1.0f);
for (LogEntry logEntry : logEntries) {
- currentLogs.add(new DfsLogger(tabletServer.getContext(), tabletServer.getServerConfig(),
- logEntry.filename, logEntry.getColumnQualifier().toString()));
+ currentLogs.add(DfsLogger.fromLogEntry(logEntry));
}
rebuildReferencedLogs();