You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by cs...@apache.org on 2023/06/23 13:23:04 UTC
[accumulo] branch no-chop-merge updated (d03972d95a -> 7fbb262b77)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a change to branch no-chop-merge
in repository https://gitbox.apache.org/repos/asf/accumulo.git
from d03972d95a Merge branch 'main' into no-chop-merge
add 3f491a4885 StoredTabletFile now lazy loads ReferencedTabletFile (#3502)
add 04b614c35f Merge branch 'main' into no-chop-merge
add fbb54c371e makes util.Pair instance variables private
add 85e1e25c89 Merge remote-tracking branch 'upstream/2.1'
add 9edc2b7cd7 [maven-release-plugin] prepare release rel/2.1.1
add 8096e0f5ea [maven-release-plugin] prepare for next development iteration
add b759a14912 Merge branch '2.1.1-rc2-next' into 2.1
add 982ee88ff3 Merge branch '2.1'
add f3ef6aca07 Revert "StoredTabletFile now lazy loads ReferencedTabletFile (#3502)"
add 2fbb779908 Avoid calling Path.getParent in TabletFile constructor (#3509)
add 847654d6a6 Merge branch '2.1' and fix conflicts
add 06c55fbea4 Create Gson singleton to use in place of new instances (#3516)
add 0214c6f99f Add missing validation to UnreferencedTabletFile
new 7fbb262b77 Merge branch 'main' into no-chop-merge
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../core/clientImpl/bulk/BulkSerialize.java | 3 +-
.../apache/accumulo/core/lock/ServiceLockData.java | 11 +-
.../accumulo/core/metadata/AbstractTabletFile.java | 8 --
.../core/metadata/ReferencedTabletFile.java | 153 +++++++++++++++++----
.../accumulo/core/metadata/StoredTabletFile.java | 5 +
.../accumulo/core/metadata/TabletDirectory.java | 65 ---------
.../core/metadata/UnreferencedTabletFile.java | 9 ++
.../accumulo/core/metadata/ValidationUtil.java | 1 +
.../schema/ExternalCompactionFinalState.java | 8 +-
.../schema/ExternalCompactionMetadata.java | 9 +-
.../core/metadata/schema/RootTabletMetadata.java | 8 +-
.../spi/compaction/DefaultCompactionPlanner.java | 6 +-
.../spi/scan/ConfigurableScanServerSelector.java | 7 +-
.../apache/accumulo/core/util/ComparablePair.java | 6 +-
.../util/{Interner.java => GsonSingleton.java} | 34 +++--
.../java/org/apache/accumulo/core/util/Pair.java | 4 +-
.../metadata/schema/ReferencedTabletFileTest.java | 6 +-
pom.xml | 2 +-
.../accumulo/server/metadata/RootGcCandidates.java | 8 +-
.../accumulo/manager/tableOps/TraceRepo.java | 5 +-
.../util/logging/AccumuloMonitorAppender.java | 6 +-
.../accumulo/test/CountNameNodeOpsBulkIT.java | 5 +-
22 files changed, 194 insertions(+), 175 deletions(-)
delete mode 100644 core/src/main/java/org/apache/accumulo/core/metadata/TabletDirectory.java
copy core/src/main/java/org/apache/accumulo/core/util/{Interner.java => GsonSingleton.java} (55%)
[accumulo] 01/01: Merge branch 'main' into no-chop-merge
Posted by cs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch no-chop-merge
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 7fbb262b77a7dcee86af169db27e90cbd28a6742
Merge: 04b614c35f 0214c6f99f
Author: Christopher L. Shannon (cshannon) <ch...@gmail.com>
AuthorDate: Fri Jun 23 09:19:30 2023 -0400
Merge branch 'main' into no-chop-merge
.../core/clientImpl/bulk/BulkSerialize.java | 3 +-
.../apache/accumulo/core/lock/ServiceLockData.java | 11 +-
.../accumulo/core/metadata/AbstractTabletFile.java | 7 -
.../core/metadata/ReferencedTabletFile.java | 156 ++++++++++++++++-----
.../accumulo/core/metadata/StoredTabletFile.java | 18 +--
.../accumulo/core/metadata/TabletDirectory.java | 65 ---------
.../core/metadata/UnreferencedTabletFile.java | 10 +-
.../accumulo/core/metadata/ValidationUtil.java | 1 +
.../schema/ExternalCompactionFinalState.java | 8 +-
.../schema/ExternalCompactionMetadata.java | 9 +-
.../core/metadata/schema/RootTabletMetadata.java | 8 +-
.../spi/compaction/DefaultCompactionPlanner.java | 6 +-
.../spi/scan/ConfigurableScanServerSelector.java | 7 +-
.../apache/accumulo/core/util/ComparablePair.java | 6 +-
.../{ComparablePair.java => GsonSingleton.java} | 30 ++--
.../java/org/apache/accumulo/core/util/Pair.java | 4 +-
.../metadata/schema/ReferencedTabletFileTest.java | 6 +-
pom.xml | 2 +-
.../accumulo/server/metadata/RootGcCandidates.java | 8 +-
.../accumulo/manager/tableOps/TraceRepo.java | 5 +-
.../util/logging/AccumuloMonitorAppender.java | 6 +-
.../accumulo/test/CountNameNodeOpsBulkIT.java | 5 +-
22 files changed, 201 insertions(+), 180 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/metadata/AbstractTabletFile.java
index c6d1ded80d,146c2b2467..51b8f6166f
--- a/core/src/main/java/org/apache/accumulo/core/metadata/AbstractTabletFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/AbstractTabletFile.java
@@@ -32,21 -31,12 +32,14 @@@ import org.apache.hadoop.fs.Path
public abstract class AbstractTabletFile<T extends AbstractTabletFile<T>>
implements TabletFile, Comparable<T> {
- private final String fileName; // C0004.rf
protected final Path path;
+ protected final Range range;
- protected AbstractTabletFile(Path path) {
+ protected AbstractTabletFile(Path path, Range range) {
this.path = Objects.requireNonNull(path);
- this.fileName = path.getName();
+ this.range = Objects.requireNonNull(range);
}
- @Override
- public String getFileName() {
- return fileName;
- }
-
@Override
public Path getPath() {
return path;
diff --cc core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java
index b1cefc5f45,ff1bfe7e3a..3edfd1d41f
--- a/core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java
@@@ -20,11 -20,11 +20,13 @@@ package org.apache.accumulo.core.metada
import static org.apache.accumulo.core.Constants.HDFS_TABLES_DIR;
+ import java.net.URI;
+import java.util.Comparator;
import java.util.Objects;
+import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
+ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
@@@ -43,48 -43,122 +45,130 @@@ import com.google.common.base.Precondit
* in Upgrader9to10.upgradeRelativePaths()
*/
public class ReferencedTabletFile extends AbstractTabletFile<ReferencedTabletFile> {
- // parts of an absolute URI, like "hdfs://1.2.3.4/accumulo/tables/2a/t-0003/C0004.rf"
- private final TabletDirectory tabletDir; // hdfs://1.2.3.4/accumulo/tables/2a/t-0003
- private final String normalizedPath;
+
+ public static class FileParts {
+
+ // parts of an absolute URI, like "hdfs://1.2.3.4/accumulo/tables/2a/t-0003/C0004.rf"
+ // volume: hdfs://1.2.3.4/accumulo
+ // tableId: 2a
+ // tabletDir: t-0003
+ // fileName: C0004.rf
+ // normalizedPath: hdfs://1.2.3.4/accumulo/tables/2a/t-0003/C0004.rf
+ private final String volume;
+ private final TableId tableId;
+ private final String tabletDir;
+ private final String fileName;
+ private final String normalizedPath;
+
+ public FileParts(String volume, TableId tableId, String tabletDir, String fileName,
+ String normalizedPath) {
+ this.volume = volume;
+ this.tableId = tableId;
+ this.tabletDir = tabletDir;
+ this.fileName = fileName;
+ this.normalizedPath = normalizedPath;
+ }
+
+ public String getVolume() {
+ return volume;
+ }
+
+ public TableId getTableId() {
+ return tableId;
+ }
+
+ public String getTabletDir() {
+ return tabletDir;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public String getNormalizedPath() {
+ return normalizedPath;
+ }
+
+ }
+
+ private static String constructErrorMsg(Path filePath) {
+ return "Missing or invalid part of tablet file metadata entry: " + filePath;
+ }
+
+ public static FileParts parsePath(Path filePath) {
+ // File name construct: <volume>/<tablePath>/<tableId>/<tablet>/<file>
+ // Example: hdfs://namenode:9020/accumulo/tables/1/default_tablet/F00001.rf
+ final URI uri = filePath.toUri();
+
+ // validate that this is a fully qualified uri
+ Preconditions.checkArgument(uri.getScheme() != null, constructErrorMsg(filePath));
+
+ final String path = uri.getPath(); // ex: /accumulo/tables/1/default_tablet/F00001.rf
+ final String[] parts = path.split("/");
+ final int numParts = parts.length; // should contain tables, 1, default_tablet, F00001.rf
+
+ if (numParts < 4) {
+ throw new IllegalArgumentException(constructErrorMsg(filePath));
+ }
+
+ final String fileName = parts[numParts - 1];
+ final String tabletDirectory = parts[numParts - 2];
+ final TableId tableId = TableId.of(parts[numParts - 3]);
+ final String tablesPath = parts[numParts - 4];
+
+ // determine where file path starts, the rest is the volume
+ final String computedFilePath =
+ HDFS_TABLES_DIR + "/" + tableId.canonical() + "/" + tabletDirectory + "/" + fileName;
+ final String uriString = uri.toString();
+ int idx = uriString.lastIndexOf(computedFilePath);
+
+ if (idx == -1) {
+ throw new IllegalArgumentException(constructErrorMsg(filePath));
+ }
+
+ // The volume is the beginning portion of the uri up to the start
+ // of the file path.
+ final String volume = uriString.substring(0, idx);
+
+ if (StringUtils.isBlank(fileName) || StringUtils.isBlank(tabletDirectory)
+ || StringUtils.isBlank(tablesPath) || StringUtils.isBlank(volume)) {
+ throw new IllegalArgumentException(constructErrorMsg(filePath));
+ }
+ ValidationUtil.validateFileName(fileName);
+ Preconditions.checkArgument(tablesPath.equals(HDFS_TABLES_DIR_NAME),
+ "tables directory name is not " + HDFS_TABLES_DIR_NAME + ", is " + tablesPath);
+
+ final String normalizedPath = volume + computedFilePath;
+
+ if (!normalizedPath.equals(uriString)) {
+ throw new RuntimeException("Error parsing file path, " + normalizedPath + " != " + uriString);
+ }
+
+ return new FileParts(volume, tableId, tabletDirectory, fileName, normalizedPath);
+
+ }
+
+ private final FileParts parts;
private static final Logger log = LoggerFactory.getLogger(ReferencedTabletFile.class);
+ private static final String HDFS_TABLES_DIR_NAME = HDFS_TABLES_DIR.substring(1);
+ private static final Comparator<ReferencedTabletFile> comparator =
+ Comparator.comparing(ReferencedTabletFile::getNormalizedPathStr)
+ .thenComparing(ReferencedTabletFile::getRange);
+
+ public ReferencedTabletFile(Path metaPath) {
+ this(metaPath, new Range());
+ }
+
/**
* Construct new tablet file using a Path. Used in the case where we had to use Path object to
* qualify an absolute path or create a new file.
*/
- public ReferencedTabletFile(Path metaPath) {
- super(Objects.requireNonNull(metaPath));
+ public ReferencedTabletFile(Path metaPath, Range range) {
+ super(Objects.requireNonNull(metaPath), range);
- String errorMsg = "Missing or invalid part of tablet file metadata entry: " + metaPath;
log.trace("Parsing TabletFile from {}", metaPath);
-
- // Validate characters in file name
- ValidationUtil.validateFileName(path.getName());
-
- // use Path object to step backwards from the filename through all the parts
- Path tabletDirPath = Objects.requireNonNull(metaPath.getParent(), errorMsg);
-
- Path tableIdPath = Objects.requireNonNull(tabletDirPath.getParent(), errorMsg);
- var id = tableIdPath.getName();
-
- Path tablePath = Objects.requireNonNull(tableIdPath.getParent(), errorMsg);
- String tpString = "/" + tablePath.getName();
- Preconditions.checkArgument(tpString.equals(HDFS_TABLES_DIR), errorMsg);
-
- Path volumePath = Objects.requireNonNull(tablePath.getParent(), errorMsg);
- Preconditions.checkArgument(volumePath.toUri().getScheme() != null, errorMsg);
- var volume = volumePath.toString();
-
- this.tabletDir = new TabletDirectory(volume, TableId.of(id), tabletDirPath.getName());
- this.normalizedPath = tabletDir.getNormalizedPath() + "/" + getFileName();
+ parts = parsePath(metaPath);
}
public String getVolume() {
@@@ -141,14 -220,14 +230,15 @@@
public boolean equals(Object obj) {
if (obj instanceof ReferencedTabletFile) {
ReferencedTabletFile that = (ReferencedTabletFile) obj;
- return normalizedPath.equals(that.normalizedPath) && range.equals(that.range);
- return parts.getNormalizedPath().equals(that.parts.getNormalizedPath());
++ return parts.getNormalizedPath().equals(that.parts.getNormalizedPath())
++ && range.equals(that.range);
}
return false;
}
@Override
public int hashCode() {
- return Objects.hash(normalizedPath, range);
- return parts.getNormalizedPath().hashCode();
++ return Objects.hash(parts.getNormalizedPath(), range);
}
@Override
diff --cc core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
index 68f84e5a5b,3bf5e701d5..feb3fd03e9
--- a/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
@@@ -19,9 -19,7 +19,8 @@@
package org.apache.accumulo.core.metadata;
import java.util.Objects;
- import java.util.function.Supplier;
+import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
@@@ -48,11 -44,9 +45,11 @@@ public class StoredTabletFile extends A
* the entry can be deleted.
*/
public StoredTabletFile(String metadataEntry) {
- super(new Path(metadataEntry));
+ // TODO: Future version of metadataEntry will contains the path
+ // and the range so we will need to parse the string here
+ super(new Path(metadataEntry), new Range());
this.metadataEntry = metadataEntry;
- this.referencedTabletFile = Suppliers.memoize(() -> ReferencedTabletFile.of(getPath()));
+ this.referencedTabletFile = ReferencedTabletFile.of(getPath());
}
/**
diff --cc core/src/main/java/org/apache/accumulo/core/metadata/UnreferencedTabletFile.java
index 4b39023fc1,4f6b16a946..3d5c1d40f8
--- a/core/src/main/java/org/apache/accumulo/core/metadata/UnreferencedTabletFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/UnreferencedTabletFile.java
@@@ -41,13 -40,17 +41,21 @@@ import org.apache.hadoop.fs.Path
*/
public class UnreferencedTabletFile extends AbstractTabletFile<UnreferencedTabletFile> {
+ private final String fileName; // C0004.rf
+
public UnreferencedTabletFile(FileSystem fs, Path path) {
- super(Objects.requireNonNull(fs).makeQualified(Objects.requireNonNull(path)));
+ this(fs, path, new Range());
+ }
+
+ public UnreferencedTabletFile(FileSystem fs, Path path, Range range) {
+ super(Objects.requireNonNull(fs).makeQualified(Objects.requireNonNull(path)), range);
- ValidationUtil.validateFileName(path.getName());
+ this.fileName = path.getName();
+ ValidationUtil.validateFileName(fileName);
+ }
+
+ @Override
+ public String getFileName() {
+ return fileName;
}
@Override