You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by dd...@apache.org on 2024/07/26 02:39:35 UTC
(accumulo) branch main updated (66f49ef8c3 -> 1a5368e983)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
from 66f49ef8c3 Merge branch '2.1'
add fd3e837033 Replace ReferenceDirectory with static named constructor (#4741)
new 1a5368e983 Merge branch '2.1'
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../accumulo/core/gc/ReferenceDirectory.java | 56 ----------------------
.../org/apache/accumulo/core/gc/ReferenceFile.java | 23 ++++++---
.../accumulo/server/gc/AllVolumesDirectory.java | 2 +-
.../main/java/org/apache/accumulo/gc/GCRun.java | 3 +-
.../accumulo/gc/GarbageCollectionAlgorithm.java | 6 +--
.../apache/accumulo/gc/GarbageCollectionTest.java | 3 +-
6 files changed, 21 insertions(+), 72 deletions(-)
delete mode 100644 core/src/main/java/org/apache/accumulo/core/gc/ReferenceDirectory.java
(accumulo) 01/01: Merge branch '2.1'
Posted by dd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 1a5368e9834aa3089948a133398036279f782d14
Merge: 66f49ef8c3 fd3e837033
Author: Daniel Roberts <dd...@gmail.com>
AuthorDate: Fri Jul 26 02:17:52 2024 +0000
Merge branch '2.1'
.../accumulo/core/gc/ReferenceDirectory.java | 56 ----------------------
.../org/apache/accumulo/core/gc/ReferenceFile.java | 23 ++++++---
.../accumulo/server/gc/AllVolumesDirectory.java | 2 +-
.../main/java/org/apache/accumulo/gc/GCRun.java | 3 +-
.../accumulo/gc/GarbageCollectionAlgorithm.java | 6 +--
.../apache/accumulo/gc/GarbageCollectionTest.java | 3 +-
6 files changed, 21 insertions(+), 72 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/gc/ReferenceFile.java
index 3044ec2f67,8a2b63cfcb..492e67c2e4
--- a/core/src/main/java/org/apache/accumulo/core/gc/ReferenceFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/gc/ReferenceFile.java
@@@ -21,9 -21,7 +21,10 @@@ package org.apache.accumulo.core.gc
import java.util.Objects;
import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.metadata.ScanServerRefTabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+ import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.hadoop.fs.Path;
/**
* A GC reference used for streaming and delete markers. This type is a file. Subclass is a
@@@ -33,34 -31,30 +34,42 @@@ public class ReferenceFile implements R
// parts of an absolute URI, like "hdfs://1.2.3.4/accumulo/tables/2a/t-0003"
public final TableId tableId; // 2a
public final boolean isScan;
+ public final boolean isDirectory;
- // the exact string that is stored in the metadata
- protected final String metadataEntry;
+ // the exact path from the file reference string that is stored in the metadata
+ protected final String metadataPath;
- protected ReferenceFile(TableId tableId, String metadataPath, boolean isScan) {
- protected ReferenceFile(TableId tableId, String metadataEntry, boolean isScan,
++ protected ReferenceFile(TableId tableId, String metadataPath, boolean isScan,
+ boolean isDirectory) {
this.tableId = Objects.requireNonNull(tableId);
- this.metadataEntry = Objects.requireNonNull(metadataEntry);
+ this.metadataPath = Objects.requireNonNull(metadataPath);
this.isScan = isScan;
+ this.isDirectory = isDirectory;
}
- public static ReferenceFile forFile(TableId tableId, String metadataEntry) {
- return new ReferenceFile(tableId, metadataEntry, false, false);
+ public static ReferenceFile forFile(TableId tableId, StoredTabletFile tabletFile) {
- return new ReferenceFile(tableId, tabletFile.getMetadataPath(), false);
++ return new ReferenceFile(tableId, tabletFile.getMetadataPath(), false, false);
}
- public static ReferenceFile forScan(TableId tableId, String metadataEntry) {
- return new ReferenceFile(tableId, metadataEntry, true, false);
+ public static ReferenceFile forFile(TableId tableId, Path metadataPathPath) {
- return new ReferenceFile(tableId, metadataPathPath.toString(), false);
++ return new ReferenceFile(tableId, metadataPathPath.toString(), false, false);
+ }
+
+ public static ReferenceFile forScan(TableId tableId, ScanServerRefTabletFile tabletFile) {
- return new ReferenceFile(tableId, tabletFile.getNormalizedPathStr(), true);
++ return new ReferenceFile(tableId, tabletFile.getNormalizedPathStr(), true, false);
+ }
+
+ public static ReferenceFile forScan(TableId tableId, StoredTabletFile tabletFile) {
- return new ReferenceFile(tableId, tabletFile.getMetadataPath(), true);
++ return new ReferenceFile(tableId, tabletFile.getMetadataPath(), true, false);
+ }
+
+ public static ReferenceFile forScan(TableId tableId, Path metadataPathPath) {
- return new ReferenceFile(tableId, metadataPathPath.toString(), true);
++ return new ReferenceFile(tableId, metadataPathPath.toString(), true, false);
+ }
+
+ public static ReferenceFile forDirectory(TableId tableId, String dirName) {
+ MetadataSchema.TabletsSection.ServerColumnFamily.validateDirCol(dirName);
+ return new ReferenceFile(tableId, dirName, false, true);
}
@Override
diff --cc server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
index aff958b010,a4b328438f..4e85fc0a7b
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GCRun.java
@@@ -50,10 -51,9 +50,9 @@@ import org.apache.accumulo.core.data.Ta
import org.apache.accumulo.core.fate.zookeeper.ZooReader;
import org.apache.accumulo.core.gc.GcCandidate;
import org.apache.accumulo.core.gc.Reference;
- import org.apache.accumulo.core.gc.ReferenceDirectory;
import org.apache.accumulo.core.gc.ReferenceFile;
import org.apache.accumulo.core.manager.state.tables.TableState;
-import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.ValidationUtil;
diff --cc server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
index eb6b8fe281,419d10728f..0c2a23b108
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectionAlgorithm.java
@@@ -143,57 -146,54 +142,56 @@@ public class GarbageCollectionAlgorith
List<GcCandidate> candidateEntriesToBeDeleted = new ArrayList<>();
Set<TableId> tableIdsBefore = gce.getCandidateTableIDs();
Set<TableId> tableIdsSeen = new HashSet<>();
- Iterator<Reference> iter = gce.getReferences().iterator();
- while (iter.hasNext()) {
- Reference ref = iter.next();
- tableIdsSeen.add(ref.getTableId());
+ try (Stream<Reference> references = gce.getReferences()) {
+ references.forEach(ref -> {
+ tableIdsSeen.add(ref.getTableId());
- if (ref.isDirectory()) {
- ServerColumnFamily.validateDirCol(ref.getMetadataEntry());
+ if (ref.isDirectory()) {
- var dirReference = (ReferenceDirectory) ref;
- ServerColumnFamily.validateDirCol(dirReference.getTabletDir());
++ ServerColumnFamily.validateDirCol(ref.getMetadataPath());
- String dir = "/" + dirReference.tableId + "/" + dirReference.getTabletDir();
- String dir = "/" + ref.getTableId() + "/" + ref.getMetadataEntry();
++ String dir = "/" + ref.getTableId() + "/" + ref.getMetadataPath();
- dir = makeRelative(dir, 2);
+ dir = makeRelative(dir, 2);
- GcCandidate gcTemp = candidateMap.remove(dir);
- if (gcTemp != null) {
- log.debug("Directory Candidate was still in use by dir ref: {}", dir);
- // Do not add dir candidates to candidateEntriesToBeDeleted as they are only created once.
- }
- } else {
- String reference = ref.getMetadataEntry();
- if (reference.startsWith("/")) {
- log.debug("Candidate {} has a relative path, prepend tableId {}", reference,
- ref.getTableId());
- reference = "/" + ref.getTableId() + ref.getMetadataEntry();
- } else if (!reference.contains(":") && !reference.startsWith("../")) {
- throw new RuntimeException("Bad file reference " + reference);
- }
+ GcCandidate gcTemp = candidateMap.remove(dir);
+ if (gcTemp != null) {
+ log.debug("Directory Candidate was still in use by dir ref: {}", dir);
+ // Do not add dir candidates to candidateEntriesToBeDeleted as they are only created
+ // once.
+ }
+ } else {
+ String reference = ref.getMetadataPath();
+ if (reference.startsWith("/")) {
+ log.debug("Candidate {} has a relative path, prepend tableId {}", reference,
+ ref.getTableId());
+ reference = "/" + ref.getTableId() + ref.getMetadataPath();
+ } else if (!reference.contains(":") && !reference.startsWith("../")) {
+ throw new RuntimeException("Bad file reference " + reference);
+ }
- String relativePath = makeRelative(reference, 3);
-
- // WARNING: This line is EXTREMELY IMPORTANT.
- // You MUST REMOVE candidates that are still in use
- GcCandidate gcTemp = candidateMap.remove(relativePath);
- if (gcTemp != null) {
- log.debug("File Candidate was still in use: {}", relativePath);
- // Prevent deletion of candidates that are still in use by scans, because they won't be
- // recreated once the scan is finished.
- if (!ref.isScan()) {
- candidateEntriesToBeDeleted.add(gcTemp);
+ String relativePath = makeRelative(reference, 3);
+
+ // WARNING: This line is EXTREMELY IMPORTANT.
+ // You MUST REMOVE candidates that are still in use
+ GcCandidate gcTemp = candidateMap.remove(relativePath);
+ if (gcTemp != null) {
+ log.debug("File Candidate was still in use: {}", relativePath);
+ // Prevent deletion of candidates that are still in use by scans, because they won't be
+ // recreated once the scan is finished.
+ if (!ref.isScan()) {
+ candidateEntriesToBeDeleted.add(gcTemp);
+ }
}
- }
- String dir = relativePath.substring(0, relativePath.lastIndexOf('/'));
- GcCandidate gcT = candidateMap.remove(dir);
- if (gcT != null) {
- log.debug("Directory Candidate was still in use by file ref: {}", relativePath);
- // Do not add dir candidates to candidateEntriesToBeDeleted as they are only created once.
+ String dir = relativePath.substring(0, relativePath.lastIndexOf('/'));
+ GcCandidate gcT = candidateMap.remove(dir);
+ if (gcT != null) {
+ log.debug("Directory Candidate was still in use by file ref: {}", relativePath);
+ // Do not add dir candidates to candidateEntriesToBeDeleted as they are only created
+ // once.
+ }
}
- }
+ });
}
Set<TableId> tableIdsAfter = gce.getCandidateTableIDs();
ensureAllTablesChecked(Collections.unmodifiableSet(tableIdsBefore),
diff --cc server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
index 4c459f150c,74b6318029..bc21ad5b35
--- a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java
@@@ -40,13 -41,13 +40,12 @@@ import org.apache.accumulo.core.client.
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.gc.GcCandidate;
import org.apache.accumulo.core.gc.Reference;
- import org.apache.accumulo.core.gc.ReferenceDirectory;
import org.apache.accumulo.core.gc.ReferenceFile;
import org.apache.accumulo.core.manager.state.tables.TableState;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.metadata.schema.Ample;
import org.apache.accumulo.core.metadata.schema.Ample.GcCandidateType;
-import org.apache.accumulo.server.replication.proto.Replication.Status;
+import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.Test;
public class GarbageCollectionTest {