You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mm...@apache.org on 2020/02/27 18:38:14 UTC
[accumulo] branch master updated: Create StoredTabletFile that
extends TabletFile. Closes #1507 (#1519)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/master by this push:
new 94844b3 Create StoredTabletFile that extends TabletFile. Closes #1507 (#1519)
94844b3 is described below
commit 94844b3ef88bc9e02b5cb5c8eeff81411ac8221d
Author: Mike Miller <mm...@apache.org>
AuthorDate: Thu Feb 27 13:38:06 2020 -0500
Create StoredTabletFile that extends TabletFile. Closes #1507 (#1519)
* Create StoredTabletFile that extends TabletFile for situations where we need to
update or delete a TabletFile or read it directly from metadata
* Created methods for read/insert/updateDel and have them called
throughout the code for the appropriate situation
* Call insert() method on TabletFile to return StoredTableFile when a
file is new and inserted
* Drop FileUtil.toPathStrings() and use object collections instead
* Replace String path with TabletFile in FileUtil
---
.../accumulo/core/clientImpl/OfflineIterator.java | 7 +-
.../accumulo/core/clientImpl/Translator.java | 10 +-
.../accumulo/core/clientImpl/Translators.java | 2 +
.../accumulo/core/metadata/StoredTabletFile.java | 62 ++++++++++++
.../apache/accumulo/core/metadata/TabletFile.java | 59 +++++------
.../accumulo/core/metadata/TabletFileUtil.java | 15 +--
.../accumulo/core/metadata/schema/Ample.java | 7 +-
.../core/metadata/schema/TabletMetadata.java | 22 ++--
.../org/apache/accumulo/core/summary/Gatherer.java | 12 +--
.../core/metadata/schema/TabletFileTest.java | 7 +-
.../core/metadata/schema/TabletMetadataTest.java | 21 ++--
.../org/apache/accumulo/server/fs/FileRef.java | 5 -
.../org/apache/accumulo/server/fs/VolumeUtil.java | 15 +--
.../accumulo/server/metadata/RootGcCandidates.java | 4 +-
.../accumulo/server/metadata/ServerAmpleImpl.java | 10 +-
.../server/metadata/TabletMutatorBase.java | 15 +--
.../org/apache/accumulo/server/util/FileUtil.java | 67 ++++++------
.../accumulo/server/util/ListVolumesUsed.java | 3 +-
.../accumulo/server/util/MasterMetadataUtil.java | 31 +++---
.../accumulo/server/util/MetadataTableUtil.java | 50 +++++----
.../apache/accumulo/server/util/FileUtilTest.java | 26 -----
.../apache/accumulo/gc/SimpleGarbageCollector.java | 2 +-
.../master/tableOps/bulkVer2/LoadFiles.java | 2 +-
.../accumulo/master/upgrade/Upgrader9to10.java | 2 +-
.../org/apache/accumulo/tserver/FileManager.java | 7 +-
.../tserver/TabletIteratorEnvironment.java | 3 +-
.../org/apache/accumulo/tserver/TabletServer.java | 8 +-
.../tserver/TabletServerResourceManager.java | 4 +-
.../tserver/compaction/CompactionPlan.java | 9 +-
.../compaction/DefaultCompactionStrategy.java | 19 ++--
.../tserver/compaction/MajorCompactionRequest.java | 11 +-
.../compaction/SizeLimitCompactionStrategy.java | 6 +-
.../strategies/BasicCompactionStrategy.java | 8 +-
.../strategies/ConfigurableCompactionStrategy.java | 21 ++--
.../TooManyDeletesCompactionStrategy.java | 4 +-
.../accumulo/tserver/tablet/CompactionInfo.java | 12 +--
.../apache/accumulo/tserver/tablet/Compactor.java | 22 ++--
.../accumulo/tserver/tablet/DatafileManager.java | 103 ++++++++++---------
.../tserver/tablet/MinorCompactionTask.java | 14 +--
.../accumulo/tserver/tablet/MinorCompactor.java | 8 +-
.../org/apache/accumulo/tserver/tablet/Tablet.java | 112 +++++++++++----------
.../apache/accumulo/tserver/tablet/TabletData.java | 11 +-
.../tserver/compaction/CompactionPlanTest.java | 24 ++---
.../compaction/DefaultCompactionStrategyTest.java | 27 ++---
.../SizeLimitCompactionStrategyTest.java | 8 +-
.../tserver/compaction/SizeWindowTest.java | 7 +-
.../strategies/BasicCompactionStrategyTest.java | 26 ++---
.../ConfigurableCompactionStrategyTest.java | 6 +-
.../tserver/tablet/DatafileManagerTest.java | 19 ++--
.../accumulo/test/SizeCompactionStrategy.java | 4 +-
.../accumulo/test/TestCompactionStrategy.java | 3 +-
.../java/org/apache/accumulo/test/VolumeIT.java | 12 +--
.../apache/accumulo/test/functional/BulkNewIT.java | 2 +-
.../accumulo/test/functional/SplitRecoveryIT.java | 37 +++----
.../test/performance/scan/CollectTabletStats.java | 6 +-
55 files changed, 546 insertions(+), 473 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java
index c24f925..2bab131 100644
--- a/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java
@@ -57,6 +57,7 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator;
import org.apache.accumulo.core.iteratorsImpl.system.SystemIteratorUtil;
import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
@@ -277,7 +278,7 @@ class OfflineIterator implements Iterator<Entry<Key,Value>> {
}
private SortedKeyValueIterator<Key,Value> createIterator(KeyExtent extent,
- Collection<TabletFile> absFiles)
+ Collection<StoredTabletFile> absFiles)
throws TableNotFoundException, AccumuloException, IOException {
// TODO share code w/ tablet - ACCUMULO-1303
@@ -310,9 +311,9 @@ class OfflineIterator implements Iterator<Entry<Key,Value>> {
// TODO need to close files - ACCUMULO-1303
for (TabletFile file : absFiles) {
FileSystem fs =
- VolumeConfiguration.getVolume(file.getMetadataEntry(), conf, config).getFileSystem();
+ VolumeConfiguration.getVolume(file.getPathStr(), conf, config).getFileSystem();
FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder()
- .forFile(file.getMetadataEntry(), fs, conf, CryptoServiceFactory.newDefaultInstance())
+ .forFile(file.getPathStr(), fs, conf, CryptoServiceFactory.newDefaultInstance())
.withTableConfiguration(acuTableConf).build();
if (scannerSamplerConfigImpl != null) {
reader = reader.getSample(scannerSamplerConfigImpl);
diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/Translator.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/Translator.java
index cc46a16..e212758 100644
--- a/core/src/main/java/org/apache/accumulo/core/clientImpl/Translator.java
+++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/Translator.java
@@ -33,6 +33,7 @@ import org.apache.accumulo.core.dataImpl.thrift.TColumn;
import org.apache.accumulo.core.dataImpl.thrift.TConstraintViolationSummary;
import org.apache.accumulo.core.dataImpl.thrift.TKeyExtent;
import org.apache.accumulo.core.dataImpl.thrift.TRange;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
public abstract class Translator<IT,OT> {
@@ -103,7 +104,14 @@ public abstract class Translator<IT,OT> {
public static class TabletFileTranslator extends Translator<TabletFile,String> {
@Override
public String translate(TabletFile input) {
- return input.getMetadataEntry();
+ return input.getPathStr();
+ }
+ }
+
+ public static class StoredTabletFileTranslator extends Translator<StoredTabletFile,String> {
+ @Override
+ public String translate(StoredTabletFile input) {
+ return input.getPathStr();
}
}
diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/Translators.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/Translators.java
index 572f4b1..01afb07 100644
--- a/core/src/main/java/org/apache/accumulo/core/clientImpl/Translators.java
+++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/Translators.java
@@ -22,6 +22,7 @@ import org.apache.accumulo.core.clientImpl.Translator.CVSTranslator;
import org.apache.accumulo.core.clientImpl.Translator.ColumnTranslator;
import org.apache.accumulo.core.clientImpl.Translator.KeyExtentTranslator;
import org.apache.accumulo.core.clientImpl.Translator.RangeTranslator;
+import org.apache.accumulo.core.clientImpl.Translator.StoredTabletFileTranslator;
import org.apache.accumulo.core.clientImpl.Translator.TCVSTranslator;
import org.apache.accumulo.core.clientImpl.Translator.TKeyExtentTranslator;
import org.apache.accumulo.core.clientImpl.Translator.TRangeTranslator;
@@ -38,4 +39,5 @@ public class Translators {
public static final Translator<Range,TRange> RT = new RangeTranslator();
public static final CVSTranslator CVST = new CVSTranslator();
public static final TabletFileTranslator TFT = new TabletFileTranslator();
+ public static final StoredTabletFileTranslator STFT = new StoredTabletFileTranslator();
}
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java b/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
new file mode 100644
index 0000000..390c7e8
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.metadata;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Object representing a tablet file entry stored in the metadata table. Keeps a string of the exact
+ * entry of what is in the metadata table, which is important for updating and deleting metadata
+ * entries. If the exact string is not used, erroneous entries can pollute the metadata table. The
+ * main column qualifier used can be found:
+ * {@link org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily}
+ * The tablet file entry is also stored for scans and bulk imports.
+ * <p>
+ * As of 2.1, Tablet file paths should now be only absolute URIs with the removal of relative paths
+ * in Upgrader9to10.upgradeRelativePaths()
+ */
+public class StoredTabletFile extends TabletFile {
+ private final String metadataEntry;
+
+ /**
+ * Construct a tablet file using the string read from the metadata. Preserve the exact string so
+ * the entry can be deleted.
+ */
+ public StoredTabletFile(String metadataEntry) {
+ super(new Path(metadataEntry));
+ this.metadataEntry = metadataEntry;
+ }
+
+ /**
+ * Return the exact string that is stored in the metadata table. This is important for updating
+ * and deleting metadata entries. If the exact string is not used, erroneous entries can pollute
+ * the metadata table.
+ */
+ public String getMetaUpdateDelete() {
+ return metadataEntry;
+ }
+
+ /**
+ * Return a new Text object of {@link #getMetaUpdateDelete()}
+ */
+ public Text getMetaUpdateDeleteText() {
+ return new Text(getMetaUpdateDelete());
+ }
+}
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/TabletFile.java b/core/src/main/java/org/apache/accumulo/core/metadata/TabletFile.java
index 83a8986..1730e70 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/TabletFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/TabletFile.java
@@ -29,9 +29,11 @@ import org.apache.hadoop.io.Text;
import com.google.common.base.Preconditions;
/**
- * Object representing a tablet file entry in the metadata table. Keeps a string of the exact entry
- * of what is in the metadata table for the column qualifier of the
- * {@link org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily}
+ * Object representing a tablet file that may exist in the metadata table. This class is used for
+ * reading and opening tablet files. It is also used when inserting new tablet files. When a new
+ * file is inserted, the {@link #insert()} method is called and returns a {@link StoredTabletFile}
+ * For situations where a tablet file needs to be updated or deleted in the metadata, a
+ * {@link StoredTabletFile} is required.
* <p>
* As of 2.1, Tablet file paths should now be only absolute URIs with the removal of relative paths
* in Upgrader9to10.upgradeRelativePaths()
@@ -42,18 +44,16 @@ public class TabletFile implements Comparable<TabletFile> {
private final TableId tableId; // 2a
private final String tabletDir; // t-0003
private final String fileName; // C0004.rf
- private final String metadataEntry;
- private final Path metaPath;
+ protected final Path metaPath;
private final String normalizedPath;
/**
- * Construct a tablet file using a Path object already created. Used in the case where we had to
- * use Path object to qualify an absolute path.
+ * Construct new tablet file using a Path. Used in the case where we had to use Path object to
+ * qualify an absolute path or create a new file.
*/
- public TabletFile(Path metaPath, String originalMetaEntry) {
- this.metadataEntry = Objects.requireNonNull(originalMetaEntry);
+ public TabletFile(Path metaPath) {
this.metaPath = Objects.requireNonNull(metaPath);
- String errorMsg = "Missing or invalid part of tablet file metadata entry: " + metadataEntry;
+ String errorMsg = "Missing or invalid part of tablet file metadata entry: " + metaPath;
// use Path object to step backwards from the filename through all the parts
this.fileName = metaPath.getName();
@@ -79,14 +79,6 @@ public class TabletFile implements Comparable<TabletFile> {
+ tabletDir + "/" + fileName;
}
- /**
- * Construct a tablet file using the string read from the metadata. Preserve the exact string so
- * the entry can be deleted.
- */
- public TabletFile(String metadataEntry) {
- this(new Path(metadataEntry), metadataEntry);
- }
-
public String getVolume() {
return volume;
}
@@ -104,21 +96,32 @@ public class TabletFile implements Comparable<TabletFile> {
}
/**
- * Exact string that is stored in the metadata table
+ * Return a string for opening and reading the tablet file. Doesn't have to be exact string in
+ * metadata.
*/
- public String getMetadataEntry() {
- return metadataEntry;
+ public String getPathStr() {
+ return normalizedPath;
}
/**
- * Exact string that is stored in the metadata table but as a Text object
+ * Return a string for inserting a new tablet file.
*/
- public Text getMetadataText() {
- return new Text(metadataEntry);
+ public String getMetaInsert() {
+ return normalizedPath;
}
- public String getNormalizedPath() {
- return normalizedPath;
+ /**
+ * Return a new Text object of {@link #getMetaInsert()}
+ */
+ public Text getMetaInsertText() {
+ return new Text(getMetaInsert());
+ }
+
+ /**
+ * New file was written to metadata so return a StoredTabletFile
+ */
+ public StoredTabletFile insert() {
+ return new StoredTabletFile(normalizedPath);
}
public Path getPath() {
@@ -130,7 +133,7 @@ public class TabletFile implements Comparable<TabletFile> {
if (equals(o)) {
return 0;
} else {
- return normalizedPath.compareTo(o.getNormalizedPath());
+ return normalizedPath.compareTo(o.normalizedPath);
}
}
@@ -138,7 +141,7 @@ public class TabletFile implements Comparable<TabletFile> {
public boolean equals(Object obj) {
if (obj instanceof TabletFile) {
TabletFile that = (TabletFile) obj;
- return normalizedPath.equals(that.getNormalizedPath());
+ return normalizedPath.equals(that.normalizedPath);
}
return false;
}
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/TabletFileUtil.java b/core/src/main/java/org/apache/accumulo/core/metadata/TabletFileUtil.java
index bbf4538..b586749 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/TabletFileUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/TabletFileUtil.java
@@ -32,16 +32,17 @@ public class TabletFileUtil {
*/
public static String validate(String path) {
Path p = new Path(path);
- if (p.toUri().getScheme() == null) {
- throw new IllegalArgumentException("Invalid path provided, no scheme in " + path);
- }
- return p.toString();
+ return validate(p).toString();
}
public static Path validate(Path path) {
- if (path.toUri().getScheme() == null) {
- throw new IllegalArgumentException("Invalid path provided, no scheme in " + path);
- }
+ check(path);
return path;
}
+
+ private static void check(Path p) {
+ if (p.toUri().getScheme() == null) {
+ throw new IllegalArgumentException("Invalid path provided, no scheme in " + p);
+ }
+ }
}
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
index 43e307f..a7039d4 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
@@ -25,6 +25,7 @@ import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.LocationType;
@@ -139,7 +140,7 @@ public interface Ample {
throw new UnsupportedOperationException();
}
- default void putGcCandidates(TableId tableId, Collection<TabletFile> candidates) {
+ default void putGcCandidates(TableId tableId, Collection<StoredTabletFile> candidates) {
throw new UnsupportedOperationException();
}
@@ -189,11 +190,11 @@ public interface Ample {
public TabletMutator putFile(TabletFile path, DataFileValue dfv);
- public TabletMutator deleteFile(TabletFile path);
+ public TabletMutator deleteFile(StoredTabletFile path);
public TabletMutator putScan(TabletFile path);
- public TabletMutator deleteScan(TabletFile path);
+ public TabletMutator deleteScan(StoredTabletFile path);
public TabletMutator putCompactionId(long compactionId);
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index 75a376e..f552be7 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@ -45,6 +45,7 @@ import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily;
@@ -76,8 +77,8 @@ public class TabletMetadata {
private boolean sawOldPrevEndRow = false;
private Text endRow;
private Location location;
- private Map<TabletFile,DataFileValue> files;
- private List<TabletFile> scans;
+ private Map<StoredTabletFile,DataFileValue> files;
+ private List<StoredTabletFile> scans;
private Map<TabletFile,Long> loadedFiles;
private EnumSet<ColumnType> fetchedCols;
private KeyExtent extent;
@@ -215,12 +216,12 @@ public class TabletMetadata {
return last;
}
- public Collection<TabletFile> getFiles() {
+ public Collection<StoredTabletFile> getFiles() {
ensureFetched(ColumnType.FILES);
return files.keySet();
}
- public Map<TabletFile,DataFileValue> getFilesMap() {
+ public Map<StoredTabletFile,DataFileValue> getFilesMap() {
ensureFetched(ColumnType.FILES);
return files;
}
@@ -230,7 +231,7 @@ public class TabletMetadata {
return logs;
}
- public List<TabletFile> getScans() {
+ public List<StoredTabletFile> getScans() {
ensureFetched(ColumnType.SCANS);
return scans;
}
@@ -281,8 +282,8 @@ public class TabletMetadata {
kvBuilder = ImmutableSortedMap.naturalOrder();
}
- var filesBuilder = ImmutableMap.<TabletFile,DataFileValue>builder();
- var scansBuilder = ImmutableList.<TabletFile>builder();
+ var filesBuilder = ImmutableMap.<StoredTabletFile,DataFileValue>builder();
+ var scansBuilder = ImmutableList.<StoredTabletFile>builder();
var logsBuilder = ImmutableList.<LogEntry>builder();
final var loadedFilesBuilder = ImmutableMap.<TabletFile,Long>builder();
ByteSequence row = null;
@@ -343,10 +344,11 @@ public class TabletMetadata {
}
break;
case DataFileColumnFamily.STR_NAME:
- filesBuilder.put(new TabletFile(qual), new DataFileValue(val));
+ filesBuilder.put(new StoredTabletFile(qual), new DataFileValue(val));
break;
case BulkFileColumnFamily.STR_NAME:
- loadedFilesBuilder.put(new TabletFile(qual), BulkFileColumnFamily.getBulkLoadTid(val));
+ loadedFilesBuilder.put(new StoredTabletFile(qual),
+ BulkFileColumnFamily.getBulkLoadTid(val));
break;
case CurrentLocationColumnFamily.STR_NAME:
te.setLocationOnce(val, qual, LocationType.CURRENT);
@@ -358,7 +360,7 @@ public class TabletMetadata {
te.last = new Location(val, qual, LocationType.LAST);
break;
case ScanFileColumnFamily.STR_NAME:
- scansBuilder.add(new TabletFile(qual));
+ scansBuilder.add(new StoredTabletFile(qual));
break;
case ClonedColumnFamily.STR_NAME:
te.cloned = val;
diff --git a/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java b/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java
index 5b77cce..466e548 100644
--- a/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java
+++ b/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java
@@ -214,9 +214,9 @@ public class Gatherer {
// When no location, the approach below will consistently choose the same tserver for the
// same file (as long as the set of tservers is stable).
- int idx = Math
- .abs(Hashing.murmur3_32().hashString(entry.getKey().getNormalizedPath(), UTF_8).asInt())
- % tservers.size();
+ int idx =
+ Math.abs(Hashing.murmur3_32().hashString(entry.getKey().getPathStr(), UTF_8).asInt())
+ % tservers.size();
location = tservers.get(idx);
}
@@ -368,9 +368,9 @@ public class Gatherer {
private synchronized void initiateProcessing(ProcessedFiles previousWork) {
try {
- Predicate<TabletFile> fileSelector = file -> Math
- .abs(Hashing.murmur3_32().hashString(file.getNormalizedPath(), UTF_8).asInt()) % modulus
- == remainder;
+ Predicate<TabletFile> fileSelector =
+ file -> Math.abs(Hashing.murmur3_32().hashString(file.getPathStr(), UTF_8).asInt())
+ % modulus == remainder;
if (previousWork != null) {
fileSelector = fileSelector.and(previousWork.failedFiles::contains);
}
diff --git a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletFileTest.java b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletFileTest.java
index 64a479a..88db9da 100644
--- a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletFileTest.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.junit.Test;
@@ -29,10 +30,10 @@ public class TabletFileTest {
private TabletFile test(String metadataEntry, String volume, String tableId, String tabletDir,
String fileName) {
- TabletFile tabletFile = new TabletFile(metadataEntry);
+ StoredTabletFile tabletFile = new StoredTabletFile(metadataEntry);
assertEquals(volume, tabletFile.getVolume());
- assertEquals(metadataEntry, tabletFile.getMetadataEntry());
+ assertEquals(metadataEntry, tabletFile.getMetaUpdateDelete());
assertEquals(TableId.of(tableId), tabletFile.getTableId());
assertEquals(tabletDir, tabletFile.getTabletDir());
assertEquals(fileName, tabletFile.getFileName());
@@ -106,7 +107,7 @@ public class TabletFileTest {
String metadataEntry = uglyVolume + "/tables/" + id + "/" + dir + "/" + filename;
TabletFile uglyFile =
test(metadataEntry, "hdfs://nn.somewhere.com:86753/accumulo", id, dir, filename);
- TabletFile niceFile = new TabletFile(
+ TabletFile niceFile = new StoredTabletFile(
"hdfs://nn.somewhere.com:86753/accumulo/tables/" + id + "/" + dir + "/" + filename);
assertEquals(niceFile, uglyFile);
assertEquals(niceFile.hashCode(), uglyFile.hashCode());
diff --git a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
index ed28f2b..9efc990 100644
--- a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
@@ -38,7 +38,7 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CurrentLocationColumnFamily;
@@ -75,12 +75,12 @@ public class TabletMetadataTest {
mutation.at().family(ClonedColumnFamily.NAME).qualifier("").put("OK");
DataFileValue dfv1 = new DataFileValue(555, 23);
- TabletFile tf1 = new TabletFile("hdfs://nn1/acc/tables/1/t-0001/df1.rf");
- TabletFile tf2 = new TabletFile("hdfs://nn1/acc/tables/1/t-0001/df2.rf");
- mutation.at().family(DataFileColumnFamily.NAME).qualifier(tf1.getMetadataEntry())
+ StoredTabletFile tf1 = new StoredTabletFile("hdfs://nn1/acc/tables/1/t-0001/df1.rf");
+ StoredTabletFile tf2 = new StoredTabletFile("hdfs://nn1/acc/tables/1/t-0001/df2.rf");
+ mutation.at().family(DataFileColumnFamily.NAME).qualifier(tf1.getMetaUpdateDelete())
.put(dfv1.encode());
DataFileValue dfv2 = new DataFileValue(234, 13);
- mutation.at().family(DataFileColumnFamily.NAME).qualifier(tf2.getMetadataEntry())
+ mutation.at().family(DataFileColumnFamily.NAME).qualifier(tf2.getMetaUpdateDelete())
.put(dfv2.encode());
mutation.at().family(CurrentLocationColumnFamily.NAME).qualifier("s001").put("server1:8555");
@@ -94,10 +94,10 @@ public class TabletMetadataTest {
mutation.at().family(le2.getColumnFamily()).qualifier(le2.getColumnQualifier())
.timestamp(le2.timestamp).put(le2.getValue());
- TabletFile sf1 = new TabletFile("hdfs://nn1/acc/tables/1/t-0001/sf1.rf");
- TabletFile sf2 = new TabletFile("hdfs://nn1/acc/tables/1/t-0001/sf2.rf");
- mutation.at().family(ScanFileColumnFamily.NAME).qualifier(sf1.getMetadataEntry()).put("");
- mutation.at().family(ScanFileColumnFamily.NAME).qualifier(sf2.getMetadataEntry()).put("");
+ StoredTabletFile sf1 = new StoredTabletFile("hdfs://nn1/acc/tables/1/t-0001/sf1.rf");
+ StoredTabletFile sf2 = new StoredTabletFile("hdfs://nn1/acc/tables/1/t-0001/sf2.rf");
+ mutation.at().family(ScanFileColumnFamily.NAME).qualifier(sf1.getMetaUpdateDelete()).put("");
+ mutation.at().family(ScanFileColumnFamily.NAME).qualifier(sf2.getMetaUpdateDelete()).put("");
SortedMap<Key,Value> rowMap = toRowMap(mutation);
@@ -113,7 +113,8 @@ public class TabletMetadataTest {
assertEquals(Map.of(tf1, dfv1, tf2, dfv2), tm.getFilesMap());
assertEquals(6L, tm.getFlushId().getAsLong());
assertEquals(rowMap, tm.getKeyValues());
- assertEquals(Map.of(new TabletFile(bf1), 56L, new TabletFile(bf2), 59L), tm.getLoaded());
+ assertEquals(Map.of(new StoredTabletFile(bf1), 56L, new StoredTabletFile(bf2), 59L),
+ tm.getLoaded());
assertEquals(HostAndPort.fromParts("server1", 8555), tm.getLocation().getHostAndPort());
assertEquals("s001", tm.getLocation().getSession());
assertEquals(LocationType.CURRENT, tm.getLocation().getType());
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/FileRef.java b/server/base/src/main/java/org/apache/accumulo/server/fs/FileRef.java
index 240f14f..afa7e8e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/FileRef.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/FileRef.java
@@ -18,7 +18,6 @@
*/
package org.apache.accumulo.server.fs;
-import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.Ample;
import org.apache.accumulo.server.fs.VolumeManager.FileType;
import org.apache.hadoop.fs.Path;
@@ -34,10 +33,6 @@ public class FileRef implements Ample.FileMeta, Comparable<FileRef> {
private Path fullReference; // something like hdfs://nn:9001/accumulo/tables/2/d-00000/A00001.rf
private Path suffix;
- public FileRef(TabletFile tabletFile) {
- this(tabletFile.getMetadataEntry(), tabletFile.getPath());
- }
-
public FileRef(String metaReference, Path fullReference) {
this.metaReference = metaReference;
this.fullReference = fullReference;
diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
index 9457067..3eb69a4 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java
@@ -25,6 +25,7 @@ import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.protobuf.ProtobufUtil;
@@ -113,7 +114,7 @@ public class VolumeUtil {
public static class TabletFiles {
public String dirName;
public List<LogEntry> logEntries;
- public SortedMap<TabletFile,DataFileValue> datafiles;
+ public SortedMap<StoredTabletFile,DataFileValue> datafiles;
public TabletFiles() {
logEntries = new ArrayList<>();
@@ -121,7 +122,7 @@ public class VolumeUtil {
}
public TabletFiles(String dirName, List<LogEntry> logEntries,
- SortedMap<TabletFile,DataFileValue> datafiles) {
+ SortedMap<StoredTabletFile,DataFileValue> datafiles) {
this.dirName = dirName;
this.logEntries = logEntries;
this.datafiles = datafiles;
@@ -144,7 +145,7 @@ public class VolumeUtil {
List<LogEntry> logsToRemove = new ArrayList<>();
List<LogEntry> logsToAdd = new ArrayList<>();
- List<TabletFile> filesToRemove = new ArrayList<>();
+ List<StoredTabletFile> filesToRemove = new ArrayList<>();
SortedMap<TabletFile,DataFileValue> filesToAdd = new TreeMap<>();
TabletFiles ret = new TabletFiles();
@@ -162,14 +163,14 @@ public class VolumeUtil {
}
}
- for (Entry<TabletFile,DataFileValue> entry : tabletFiles.datafiles.entrySet()) {
- String metaPath = entry.getKey().getNormalizedPath();
+ for (Entry<StoredTabletFile,DataFileValue> entry : tabletFiles.datafiles.entrySet()) {
+ String metaPath = entry.getKey().getMetaUpdateDelete();
Path switchedPath = switchVolume(metaPath, FileType.TABLE, replacements);
if (switchedPath != null) {
filesToRemove.add(entry.getKey());
- TabletFile switchedFile = new TabletFile(switchedPath, switchedPath.toString());
+ TabletFile switchedFile = new TabletFile(switchedPath);
filesToAdd.put(switchedFile, entry.getValue());
- ret.datafiles.put(switchedFile, entry.getValue());
+ ret.datafiles.put(switchedFile.insert(), entry.getValue());
log.debug("Replacing volume {} : {} -> {}", extent, metaPath, switchedPath);
} else {
ret.datafiles.put(entry.getKey(), entry.getValue());
diff --git a/server/base/src/main/java/org/apache/accumulo/server/metadata/RootGcCandidates.java b/server/base/src/main/java/org/apache/accumulo/server/metadata/RootGcCandidates.java
index 5f26f9a..53dd6c5 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/metadata/RootGcCandidates.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/RootGcCandidates.java
@@ -27,7 +27,7 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.stream.Stream;
-import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.hadoop.fs.Path;
import com.google.common.base.Preconditions;
@@ -63,7 +63,7 @@ public class RootGcCandidates {
this.candidates = candidates;
}
- public void add(Collection<TabletFile> refs) {
+ public void add(Collection<StoredTabletFile> refs) {
refs.forEach(ref -> {
String parent = ref.getPath().getParent().toString();
candidates.computeIfAbsent(parent, k -> new TreeSet<>()).add(ref.getFileName());
diff --git a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
index bfa0e28..a2f6d1f 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
@@ -40,7 +40,7 @@ import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
-import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFileUtil;
import org.apache.accumulo.core.metadata.schema.Ample;
import org.apache.accumulo.core.metadata.schema.AmpleImpl;
@@ -109,7 +109,7 @@ public class ServerAmpleImpl extends AmpleImpl implements Ample {
}
@Override
- public void putGcCandidates(TableId tableId, Collection<TabletFile> candidates) {
+ public void putGcCandidates(TableId tableId, Collection<StoredTabletFile> candidates) {
if (RootTable.ID.equals(tableId)) {
mutateRootGcCandidates(rgcc -> rgcc.add(candidates));
@@ -117,7 +117,7 @@ public class ServerAmpleImpl extends AmpleImpl implements Ample {
}
try (BatchWriter writer = createWriter(tableId)) {
- for (TabletFile file : candidates) {
+ for (StoredTabletFile file : candidates) {
writer.addMutation(createDeleteMutation(file));
}
} catch (MutationsRejectedException e) {
@@ -198,8 +198,8 @@ public class ServerAmpleImpl extends AmpleImpl implements Ample {
return createDelMutation(path);
}
- public static Mutation createDeleteMutation(TabletFile pathToRemove) {
- return createDelMutation(pathToRemove.getMetadataEntry());
+ public static Mutation createDeleteMutation(StoredTabletFile pathToRemove) {
+ return createDelMutation(pathToRemove.getMetaUpdateDelete());
}
private static Mutation createDelMutation(String path) {
diff --git a/server/base/src/main/java/org/apache/accumulo/server/metadata/TabletMutatorBase.java b/server/base/src/main/java/org/apache/accumulo/server/metadata/TabletMutatorBase.java
index cc756bc..6dd9a75 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/metadata/TabletMutatorBase.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/TabletMutatorBase.java
@@ -21,6 +21,7 @@ package org.apache.accumulo.server.metadata;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.Ample;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
@@ -72,28 +73,28 @@ public abstract class TabletMutatorBase implements Ample.TabletMutator {
@Override
public Ample.TabletMutator putFile(TabletFile path, DataFileValue dfv) {
Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
- mutation.put(DataFileColumnFamily.NAME, path.getMetadataText(), new Value(dfv.encode()));
+ mutation.put(DataFileColumnFamily.NAME, path.getMetaInsertText(), new Value(dfv.encode()));
return this;
}
@Override
- public Ample.TabletMutator deleteFile(TabletFile path) {
+ public Ample.TabletMutator deleteFile(StoredTabletFile path) {
Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
- mutation.putDelete(DataFileColumnFamily.NAME, path.getMetadataText());
+ mutation.putDelete(DataFileColumnFamily.NAME, path.getMetaUpdateDeleteText());
return this;
}
@Override
public Ample.TabletMutator putScan(TabletFile path) {
Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
- mutation.put(ScanFileColumnFamily.NAME, path.getMetadataText(), new Value(new byte[0]));
+ mutation.put(ScanFileColumnFamily.NAME, path.getMetaInsertText(), new Value(new byte[0]));
return this;
}
@Override
- public Ample.TabletMutator deleteScan(TabletFile path) {
+ public Ample.TabletMutator deleteScan(StoredTabletFile path) {
Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
- mutation.putDelete(ScanFileColumnFamily.NAME, path.getMetadataText());
+ mutation.putDelete(ScanFileColumnFamily.NAME, path.getMetaUpdateDeleteText());
return this;
}
@@ -178,7 +179,7 @@ public abstract class TabletMutatorBase implements Ample.TabletMutator {
@Override
public Ample.TabletMutator putBulkFile(TabletFile bulkref, long tid) {
Preconditions.checkState(updatesEnabled, "Cannot make updates after calling mutate.");
- mutation.put(TabletsSection.BulkFileColumnFamily.NAME, bulkref.getMetadataText(),
+ mutation.put(TabletsSection.BulkFileColumnFamily.NAME, bulkref.getMetaInsertText(),
new Value(FateTxId.formatTid(tid)));
return this;
}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
index a509dd3..bf9b401 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/FileUtil.java
@@ -112,13 +112,13 @@ public class FileUtil {
return result;
}
- public static Collection<String> reduceFiles(ServerContext context, Configuration conf,
- Text prevEndRow, Text endRow, Collection<String> mapFiles, int maxFiles, Path tmpDir,
+ public static Collection<TabletFile> reduceFiles(ServerContext context, Configuration conf,
+ Text prevEndRow, Text endRow, Collection<TabletFile> mapFiles, int maxFiles, Path tmpDir,
int pass) throws IOException {
AccumuloConfiguration acuConf = context.getConfiguration();
- ArrayList<String> paths = new ArrayList<>(mapFiles);
+ ArrayList<TabletFile> paths = new ArrayList<>(mapFiles);
if (paths.size() <= maxFiles)
return paths;
@@ -127,33 +127,34 @@ public class FileUtil {
int start = 0;
- ArrayList<String> outFiles = new ArrayList<>();
+ ArrayList<TabletFile> outFiles = new ArrayList<>();
int count = 0;
while (start < paths.size()) {
int end = Math.min(maxFiles + start, paths.size());
- List<String> inFiles = paths.subList(start, end);
+ List<TabletFile> inFiles = paths.subList(start, end);
start = end;
- String newMapFile = String.format("%s/%04d.%s", newDir, count++, RFile.EXTENSION);
+ TabletFile newMapFile =
+ new TabletFile(new Path(String.format("%s/%04d.%s", newDir, count++, RFile.EXTENSION)));
outFiles.add(newMapFile);
FileSystem ns =
- context.getVolumeManager().getVolumeByPath(new Path(newMapFile)).getFileSystem();
+ context.getVolumeManager().getVolumeByPath(newMapFile.getPath()).getFileSystem();
FileSKVWriter writer = new RFileOperations().newWriterBuilder()
- .forFile(newMapFile.toString(), ns, ns.getConf(), context.getCryptoService())
+ .forFile(newMapFile.getPathStr(), ns, ns.getConf(), context.getCryptoService())
.withTableConfiguration(acuConf).build();
writer.startDefaultLocalityGroup();
List<SortedKeyValueIterator<Key,Value>> iters = new ArrayList<>(inFiles.size());
FileSKVIterator reader = null;
try {
- for (String s : inFiles) {
- ns = context.getVolumeManager().getVolumeByPath(new Path(s)).getFileSystem();
+ for (TabletFile file : inFiles) {
+ ns = context.getVolumeManager().getVolumeByPath(file.getPath()).getFileSystem();
reader = FileOperations.getInstance().newIndexReaderBuilder()
- .forFile(s, ns, ns.getConf(), context.getCryptoService())
+ .forFile(file.getPathStr(), ns, ns.getConf(), context.getCryptoService())
.withTableConfiguration(acuConf).build();
iters.add(reader);
}
@@ -204,13 +205,14 @@ public class FileUtil {
}
public static SortedMap<Double,Key> findMidPoint(ServerContext context, String tabletDir,
- Text prevEndRow, Text endRow, Collection<String> mapFiles, double minSplit)
+ Text prevEndRow, Text endRow, Collection<TabletFile> mapFiles, double minSplit)
throws IOException {
return findMidPoint(context, tabletDir, prevEndRow, endRow, mapFiles, minSplit, true);
}
public static double estimatePercentageLTE(ServerContext context, String tabletDir,
- Text prevEndRow, Text endRow, Collection<String> mapFiles, Text splitRow) throws IOException {
+ Text prevEndRow, Text endRow, Collection<TabletFile> mapFiles, Text splitRow)
+ throws IOException {
Path tmpDir = null;
@@ -287,10 +289,10 @@ public class FileUtil {
* indexing interval is unknown.
*/
public static SortedMap<Double,Key> findMidPoint(ServerContext context, String tabletDirectory,
- Text prevEndRow, Text endRow, Collection<String> mapFiles, double minSplit, boolean useIndex)
- throws IOException {
+ Text prevEndRow, Text endRow, Collection<TabletFile> mapFiles, double minSplit,
+ boolean useIndex) throws IOException {
- Collection<String> origMapFiles = mapFiles;
+ Collection<TabletFile> origMapFiles = mapFiles;
Path tmpDir = null;
@@ -424,7 +426,7 @@ public class FileUtil {
}
private static long countIndexEntries(ServerContext context, Text prevEndRow, Text endRow,
- Collection<String> mapFiles, boolean useIndex, ArrayList<FileSKVIterator> readers)
+ Collection<TabletFile> mapFiles, boolean useIndex, ArrayList<FileSKVIterator> readers)
throws IOException {
AccumuloConfiguration acuConf = context.getConfiguration();
@@ -432,18 +434,17 @@ public class FileUtil {
long numKeys = 0;
// count the total number of index entries
- for (String ref : mapFiles) {
+ for (TabletFile file : mapFiles) {
FileSKVIterator reader = null;
- Path path = new Path(ref);
- FileSystem ns = context.getVolumeManager().getVolumeByPath(path).getFileSystem();
+ FileSystem ns = context.getVolumeManager().getVolumeByPath(file.getPath()).getFileSystem();
try {
if (useIndex)
reader = FileOperations.getInstance().newIndexReaderBuilder()
- .forFile(path.toString(), ns, ns.getConf(), context.getCryptoService())
+ .forFile(file.getPathStr(), ns, ns.getConf(), context.getCryptoService())
.withTableConfiguration(acuConf).build();
else
reader = FileOperations.getInstance().newScanReaderBuilder()
- .forFile(path.toString(), ns, ns.getConf(), context.getCryptoService())
+ .forFile(file.getPathStr(), ns, ns.getConf(), context.getCryptoService())
.withTableConfiguration(acuConf).overRange(new Range(prevEndRow, false, null, true),
LocalityGroupUtil.EMPTY_CF_SET, false)
.build();
@@ -468,11 +469,11 @@ public class FileUtil {
if (useIndex)
readers.add(FileOperations.getInstance().newIndexReaderBuilder()
- .forFile(path.toString(), ns, ns.getConf(), context.getCryptoService())
+ .forFile(file.getPathStr(), ns, ns.getConf(), context.getCryptoService())
.withTableConfiguration(acuConf).build());
else
readers.add(FileOperations.getInstance().newScanReaderBuilder()
- .forFile(path.toString(), ns, ns.getConf(), context.getCryptoService())
+ .forFile(file.getPathStr(), ns, ns.getConf(), context.getCryptoService())
.withTableConfiguration(acuConf).overRange(new Range(prevEndRow, false, null, true),
LocalityGroupUtil.EMPTY_CF_SET, false)
.build());
@@ -494,7 +495,7 @@ public class FileUtil {
FileSystem ns = context.getVolumeManager().getVolumeByPath(mapfile.getPath()).getFileSystem();
try {
reader = FileOperations.getInstance().newReaderBuilder()
- .forFile(mapfile.toString(), ns, ns.getConf(), context.getCryptoService())
+ .forFile(mapfile.getPathStr(), ns, ns.getConf(), context.getCryptoService())
.withTableConfiguration(context.getConfiguration()).build();
Key firstKey = reader.getFirstKey();
@@ -529,11 +530,10 @@ public class FileUtil {
Key lastKey = null;
- for (TabletFile ref : mapFiles) {
- Path path = ref.getPath();
- FileSystem ns = context.getVolumeManager().getVolumeByPath(path).getFileSystem();
+ for (TabletFile file : mapFiles) {
+ FileSystem ns = context.getVolumeManager().getVolumeByPath(file.getPath()).getFileSystem();
FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder()
- .forFile(path.toString(), ns, ns.getConf(), context.getCryptoService())
+ .forFile(file.getPathStr(), ns, ns.getConf(), context.getCryptoService())
.withTableConfiguration(context.getConfiguration()).seekToBeginning().build();
try {
@@ -567,13 +567,4 @@ public class FileUtil {
null, context.getCryptoService());
}
- public static Collection<String> toPathStrings(Collection<TabletFile> refs) {
- ArrayList<String> ret = new ArrayList<>();
- for (TabletFile fileRef : refs) {
- ret.add(fileRef.getNormalizedPath());
- }
-
- return ret;
- }
-
}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java b/server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java
index 27d3542..f5a836e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/ListVolumesUsed.java
@@ -65,8 +65,7 @@ public class ListVolumesUsed {
try (TabletsMetadata tablets = TabletsMetadata.builder().forLevel(level)
.fetch(TabletMetadata.ColumnType.FILES, TabletMetadata.ColumnType.LOGS).build(context)) {
for (TabletMetadata tabletMetadata : tablets) {
- tabletMetadata.getFiles()
- .forEach(file -> volumes.add(getTableURI(file.getMetadataEntry())));
+ tabletMetadata.getFiles().forEach(file -> volumes.add(getTableURI(file.getPathStr())));
tabletMetadata.getLogs().forEach(le -> getLogURIs(volumes, le));
}
}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
index c9579e1..366e941 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MasterMetadataUtil.java
@@ -41,6 +41,7 @@ import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.Ample.TabletMutator;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
@@ -63,7 +64,7 @@ public class MasterMetadataUtil {
private static final Logger log = LoggerFactory.getLogger(MasterMetadataUtil.class);
public static void addNewTablet(ServerContext context, KeyExtent extent, String dirName,
- TServerInstance location, Map<TabletFile,DataFileValue> datafileSizes,
+ TServerInstance location, Map<StoredTabletFile,DataFileValue> datafileSizes,
Map<Long,? extends Collection<TabletFile>> bulkLoadedFiles, MetadataTime time,
long lastFlushID, long lastCompactID, ZooLock zooLock) {
@@ -88,7 +89,7 @@ public class MasterMetadataUtil {
for (Entry<Long,? extends Collection<TabletFile>> entry : bulkLoadedFiles.entrySet()) {
for (TabletFile ref : entry.getValue()) {
- tablet.putBulkFile(ref, entry.getKey().longValue());
+ tablet.putBulkFile(ref, entry.getKey());
}
}
@@ -135,11 +136,11 @@ public class MasterMetadataUtil {
} else {
log.info("Finishing incomplete split {} {}", metadataEntry, metadataPrevEndRow);
- List<TabletFile> highDatafilesToRemove = new ArrayList<>();
+ List<StoredTabletFile> highDatafilesToRemove = new ArrayList<>();
- SortedMap<TabletFile,DataFileValue> origDatafileSizes = new TreeMap<>();
- SortedMap<TabletFile,DataFileValue> highDatafileSizes = new TreeMap<>();
- SortedMap<TabletFile,DataFileValue> lowDatafileSizes = new TreeMap<>();
+ SortedMap<StoredTabletFile,DataFileValue> origDatafileSizes = new TreeMap<>();
+ SortedMap<StoredTabletFile,DataFileValue> highDatafileSizes = new TreeMap<>();
+ SortedMap<StoredTabletFile,DataFileValue> lowDatafileSizes = new TreeMap<>();
try (Scanner scanner3 = new ScannerImpl(context, MetadataTable.ID, Authorizations.EMPTY)) {
Key rowKey = new Key(metadataEntry);
@@ -149,8 +150,9 @@ public class MasterMetadataUtil {
for (Entry<Key,Value> entry : scanner3) {
if (entry.getKey().compareColumnFamily(DataFileColumnFamily.NAME) == 0) {
- TabletFile tf = new TabletFile(entry.getKey().getColumnQualifierData().toString());
- origDatafileSizes.put(tf, new DataFileValue(entry.getValue().get()));
+ StoredTabletFile stf =
+ new StoredTabletFile(entry.getKey().getColumnQualifierData().toString());
+ origDatafileSizes.put(stf, new DataFileValue(entry.getValue().get()));
}
}
}
@@ -178,7 +180,7 @@ public class MasterMetadataUtil {
}
public static void replaceDatafiles(ServerContext context, KeyExtent extent,
- Set<TabletFile> datafilesToDelete, Set<TabletFile> scanFiles, TabletFile path,
+ Set<StoredTabletFile> datafilesToDelete, Set<StoredTabletFile> scanFiles, TabletFile path,
Long compactionId, DataFileValue size, String address, TServerInstance lastLocation,
ZooLock zooLock) {
@@ -214,16 +216,18 @@ public class MasterMetadataUtil {
* should be relative to the table directory
*
*/
- public static void updateTabletDataFile(ServerContext context, KeyExtent extent, TabletFile path,
- TabletFile mergeFile, DataFileValue dfv, MetadataTime time, Set<TabletFile> filesInUseByScans,
- String address, ZooLock zooLock, Set<String> unusedWalLogs, TServerInstance lastLocation,
- long flushId) {
+ public static StoredTabletFile updateTabletDataFile(ServerContext context, KeyExtent extent,
+ TabletFile path, StoredTabletFile mergeFile, DataFileValue dfv, MetadataTime time,
+ Set<StoredTabletFile> filesInUseByScans, String address, ZooLock zooLock,
+ Set<String> unusedWalLogs, TServerInstance lastLocation, long flushId) {
TabletMutator tablet = context.getAmple().mutateTablet(extent);
+ StoredTabletFile newFile = null;
if (dfv.getNumEntries() > 0) {
tablet.putFile(path, dfv);
tablet.putTime(time);
+ newFile = path.insert();
TServerInstance self = getTServerInstance(address, zooLock);
tablet.putLocation(self, LocationType.LAST);
@@ -245,5 +249,6 @@ public class MasterMetadataUtil {
tablet.putZooLock(zooLock);
tablet.mutate();
+ return newFile;
}
}
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index cadcbd8..e855821 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -64,6 +64,7 @@ import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.TabletFileUtil;
import org.apache.accumulo.core.metadata.schema.Ample.TabletMutator;
@@ -176,18 +177,21 @@ public class MetadataTableUtil {
tablet.mutate();
}
- public static void updateTabletDataFile(long tid, KeyExtent extent,
+ public static Map<StoredTabletFile,DataFileValue> updateTabletDataFile(long tid, KeyExtent extent,
Map<TabletFile,DataFileValue> estSizes, MetadataTime time, ServerContext context,
ZooLock zooLock) {
TabletMutator tablet = context.getAmple().mutateTablet(extent);
tablet.putTime(time);
- estSizes.forEach(tablet::putFile);
- for (TabletFile file : estSizes.keySet()) {
- tablet.putBulkFile(file, tid);
- }
+ Map<StoredTabletFile,DataFileValue> newFiles = new HashMap<>(estSizes.size());
+ estSizes.forEach((tf, dfv) -> {
+ tablet.putFile(tf, dfv);
+ tablet.putBulkFile(tf, tid);
+ newFiles.put(tf.insert(), dfv);
+ });
tablet.putZooLock(zooLock);
tablet.mutate();
+ return newFiles;
}
public static void updateTabletDir(KeyExtent extent, String newDir, ServerContext context,
@@ -210,7 +214,7 @@ public class MetadataTableUtil {
}
public static void updateTabletVolumes(KeyExtent extent, List<LogEntry> logsToRemove,
- List<LogEntry> logsToAdd, List<TabletFile> filesToRemove,
+ List<LogEntry> logsToAdd, List<StoredTabletFile> filesToRemove,
SortedMap<TabletFile,DataFileValue> filesToAdd, ZooLock zooLock, ServerContext context) {
TabletMutator tabletMutator = context.getAmple().mutateTablet(extent);
@@ -247,27 +251,29 @@ public class MetadataTableUtil {
update(context, zooLock, m, extent);
}
- public static void finishSplit(Text metadataEntry, Map<TabletFile,DataFileValue> datafileSizes,
- List<TabletFile> highDatafilesToRemove, final ServerContext context, ZooLock zooLock) {
+ public static void finishSplit(Text metadataEntry,
+ Map<StoredTabletFile,DataFileValue> datafileSizes,
+ List<StoredTabletFile> highDatafilesToRemove, final ServerContext context, ZooLock zooLock) {
Mutation m = new Mutation(metadataEntry);
TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.putDelete(m);
TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.putDelete(m);
ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m);
- for (Entry<TabletFile,DataFileValue> entry : datafileSizes.entrySet()) {
- m.put(DataFileColumnFamily.NAME, entry.getKey().getMetadataText(),
+ for (Entry<StoredTabletFile,DataFileValue> entry : datafileSizes.entrySet()) {
+ m.put(DataFileColumnFamily.NAME, entry.getKey().getMetaInsertText(),
new Value(entry.getValue().encode()));
}
- for (TabletFile pathToRemove : highDatafilesToRemove) {
- m.putDelete(DataFileColumnFamily.NAME, pathToRemove.getMetadataText());
+ for (StoredTabletFile pathToRemove : highDatafilesToRemove) {
+ m.putDelete(DataFileColumnFamily.NAME, pathToRemove.getMetaUpdateDeleteText());
}
update(context, zooLock, m, new KeyExtent(metadataEntry, (Text) null));
}
- public static void finishSplit(KeyExtent extent, Map<TabletFile,DataFileValue> datafileSizes,
- List<TabletFile> highDatafilesToRemove, ServerContext context, ZooLock zooLock) {
+ public static void finishSplit(KeyExtent extent,
+ Map<StoredTabletFile,DataFileValue> datafileSizes,
+ List<StoredTabletFile> highDatafilesToRemove, ServerContext context, ZooLock zooLock) {
finishSplit(extent.getMetadataEntry(), datafileSizes, highDatafilesToRemove, context, zooLock);
}
@@ -288,7 +294,7 @@ public class MetadataTableUtil {
update(context, ServerAmpleImpl.createDeleteMutation(path), new KeyExtent(tableId, null, null));
}
- public static void removeScanFiles(KeyExtent extent, Set<TabletFile> scanFiles,
+ public static void removeScanFiles(KeyExtent extent, Set<StoredTabletFile> scanFiles,
ServerContext context, ZooLock zooLock) {
TabletMutator tablet = context.getAmple().mutateTablet(extent);
scanFiles.forEach(tablet::deleteScan);
@@ -298,12 +304,12 @@ public class MetadataTableUtil {
public static void splitDatafiles(Text midRow, double splitRatio,
Map<TabletFile,FileUtil.FileInfo> firstAndLastRows,
- SortedMap<TabletFile,DataFileValue> datafiles,
- SortedMap<TabletFile,DataFileValue> lowDatafileSizes,
- SortedMap<TabletFile,DataFileValue> highDatafileSizes,
- List<TabletFile> highDatafilesToRemove) {
+ SortedMap<StoredTabletFile,DataFileValue> datafiles,
+ SortedMap<StoredTabletFile,DataFileValue> lowDatafileSizes,
+ SortedMap<StoredTabletFile,DataFileValue> highDatafileSizes,
+ List<StoredTabletFile> highDatafilesToRemove) {
- for (Entry<TabletFile,DataFileValue> entry : datafiles.entrySet()) {
+ for (Entry<StoredTabletFile,DataFileValue> entry : datafiles.entrySet()) {
Text firstRow = null;
Text lastRow = null;
@@ -408,10 +414,10 @@ public class MetadataTableUtil {
}
}
- public static Pair<List<LogEntry>,SortedMap<TabletFile,DataFileValue>>
+ public static Pair<List<LogEntry>,SortedMap<StoredTabletFile,DataFileValue>>
getFileAndLogEntries(ServerContext context, KeyExtent extent) throws IOException {
ArrayList<LogEntry> result = new ArrayList<>();
- TreeMap<TabletFile,DataFileValue> sizes = new TreeMap<>();
+ TreeMap<StoredTabletFile,DataFileValue> sizes = new TreeMap<>();
TabletMetadata tablet = context.getAmple().readTablet(extent, FILES, LOGS, PREV_ROW, DIR);
diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/FileUtilTest.java b/server/base/src/test/java/org/apache/accumulo/server/util/FileUtilTest.java
index dba2c33..aeab364 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/util/FileUtilTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/util/FileUtilTest.java
@@ -18,22 +18,15 @@
*/
package org.apache.accumulo.server.util;
-import static org.easymock.EasyMock.createMock;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.HashMap;
-import java.util.Iterator;
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
import org.apache.hadoop.fs.Path;
@@ -65,25 +58,6 @@ public class FileUtilTest {
}
@Test
- public void testToPathStrings() {
- Collection<TabletFile> c = new java.util.ArrayList<>();
- TabletFile r1 = createMock(TabletFile.class);
- expect(r1.getNormalizedPath()).andReturn("/foo");
- replay(r1);
- c.add(r1);
- TabletFile r2 = createMock(TabletFile.class);
- expect(r2.getNormalizedPath()).andReturn("/bar");
- replay(r2);
- c.add(r2);
-
- Collection<String> cs = FileUtil.toPathStrings(c);
- assertEquals(2, cs.size());
- Iterator<String> iter = cs.iterator();
- assertEquals("/foo", iter.next());
- assertEquals("/bar", iter.next());
- }
-
- @Test
public void testCleanupIndexOpWithDfsDir() throws IOException {
// And a "unique" tmp directory for each volume
File tmp1 = new File(accumuloDir, "tmp");
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 9031e44..93a58b5 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -258,7 +258,7 @@ public class SimpleGarbageCollector extends AbstractServer implements Iface {
Stream<Reference> refStream = tabletStream.flatMap(tm -> {
Stream<Reference> refs = Stream.concat(tm.getFiles().stream(), tm.getScans().stream())
- .map(f -> new Reference(tm.getTableId(), f.getMetadataEntry(), false));
+ .map(f -> new Reference(tm.getTableId(), f.getMetaUpdateDelete(), false));
if (tm.getDirName() != null) {
refs =
Stream.concat(refs, Stream.of(new Reference(tm.getTableId(), tm.getDirName(), true)));
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java
index 36ce66c..e031a0f 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java
@@ -220,7 +220,7 @@ class LoadFiles extends MasterRepo {
for (final Bulk.FileInfo fileInfo : files) {
Path fullPath = new Path(bulkDir, fileInfo.getFileName());
- TabletFile bulkFile = new TabletFile(fullPath, fullPath.toString());
+ TabletFile bulkFile = new TabletFile(fullPath);
if (!loadedFiles.contains(bulkFile)) {
thriftImports.put(fileInfo.getFileName(), new MapFileInfo(fileInfo.getEstFileSize()));
diff --git a/server/master/src/main/java/org/apache/accumulo/master/upgrade/Upgrader9to10.java b/server/master/src/main/java/org/apache/accumulo/master/upgrade/Upgrader9to10.java
index d8a0876..09e6c62 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/upgrade/Upgrader9to10.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/upgrade/Upgrader9to10.java
@@ -154,7 +154,7 @@ public class Upgrader9to10 implements Upgrader {
logs.forEach(tabletMutator::putWal);
Map<String,DataFileValue> files = cleanupRootTabletFiles(ctx.getVolumeManager(), dir);
- files.forEach((path, dfv) -> tabletMutator.putFile(new TabletFile(path), dfv));
+ files.forEach((path, dfv) -> tabletMutator.putFile(new TabletFile(new Path(path)), dfv));
tabletMutator.putTime(computeRootTabletTime(ctx, files.keySet()));
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
index 3cb16d7..2a45236 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java
@@ -497,8 +497,9 @@ public class FileManager {
private Map<FileSKVIterator,String> openFileRefs(Collection<TabletFile> files)
throws TooManyFilesException, IOException {
List<String> strings = new ArrayList<>(files.size());
- for (TabletFile ref : files)
- strings.add(ref.getMetadataEntry());
+ for (TabletFile file : files) {
+ strings.add(file.getPathStr());
+ }
return openFiles(strings);
}
@@ -562,7 +563,7 @@ public class FileManager {
if (sawTimeSet) {
// constructing FileRef is expensive so avoid if not needed
- DataFileValue value = files.get(new TabletFile(filename));
+ DataFileValue value = files.get(new TabletFile(new Path(filename)));
if (value.isTimeSet()) {
iter = new TimeSettingIterator(iter, value.getTime());
}
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletIteratorEnvironment.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletIteratorEnvironment.java
index 69e878a..de58ddd 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletIteratorEnvironment.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletIteratorEnvironment.java
@@ -43,6 +43,7 @@ import org.apache.accumulo.server.ServiceEnvironmentImpl;
import org.apache.accumulo.server.iterators.SystemIteratorEnvironment;
import org.apache.accumulo.tserver.FileManager.ScanFileManager;
import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
+import org.apache.hadoop.fs.Path;
public class TabletIteratorEnvironment implements SystemIteratorEnvironment {
@@ -153,7 +154,7 @@ public class TabletIteratorEnvironment implements SystemIteratorEnvironment {
@Override
public SortedKeyValueIterator<Key,Value> reserveMapFileReader(String mapFileName)
throws IOException {
- TabletFile ref = new TabletFile(mapFileName);
+ TabletFile ref = new TabletFile(new Path(mapFileName));
return trm.openFiles(Collections.singletonMap(ref, files.get(ref)), false, null).get(0);
}
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index b64db13..6e1048f 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -537,7 +537,7 @@ public class TabletServer extends AbstractServer {
Path path = new Path(mapping.getKey());
FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
path = ns.makeQualified(path);
- fileRefMap.put(new TabletFile(path.toString()), mapping.getValue());
+ fileRefMap.put(new TabletFile(path), mapping.getValue());
}
Tablet importTablet = getOnlineTablet(new KeyExtent(tke));
@@ -574,19 +574,19 @@ public class TabletServer extends AbstractServer {
watcher.runQuietly(Constants.BULK_ARBITRATOR_TYPE, tid, () -> {
tabletImports.forEach((tke, fileMap) -> {
- Map<TabletFile,MapFileInfo> fileRefMap = new HashMap<>();
+ Map<TabletFile,MapFileInfo> newFileMap = new HashMap<>();
for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) {
Path path = new Path(dir, mapping.getKey());
FileSystem ns = fs.getVolumeByPath(path).getFileSystem();
path = ns.makeQualified(path);
- fileRefMap.put(new TabletFile(path, path.toString()), mapping.getValue());
+ newFileMap.put(new TabletFile(path), mapping.getValue());
}
Tablet importTablet = getOnlineTablet(new KeyExtent(tke));
if (importTablet != null) {
try {
- importTablet.importMapFiles(tid, fileRefMap, setTime);
+ importTablet.importMapFiles(tid, newFileMap, setTime);
} catch (IOException ioe) {
log.debug("files {} not imported to {}: {}", fileMap.keySet(), new KeyExtent(tke),
ioe.getMessage());
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
index dcd42a5..41a748b 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
@@ -54,7 +54,7 @@ import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.file.blockfile.cache.impl.BlockCacheConfiguration;
import org.apache.accumulo.core.file.blockfile.cache.impl.BlockCacheManagerFactory;
import org.apache.accumulo.core.file.blockfile.impl.ScanCacheProvider;
-import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.spi.cache.BlockCache;
import org.apache.accumulo.core.spi.cache.BlockCacheManager;
@@ -824,7 +824,7 @@ public class TabletServerResourceManager {
// BEGIN methods that Tablets call to make decisions about major compaction
// when too many files are open, we may want tablets to compact down
// to one map file
- public boolean needsMajorCompaction(SortedMap<TabletFile,DataFileValue> tabletFiles,
+ public boolean needsMajorCompaction(SortedMap<StoredTabletFile,DataFileValue> tabletFiles,
MajorCompactionReason reason) {
if (closed) {
return false;// throw new IOException("closed");
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/CompactionPlan.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/CompactionPlan.java
index e1d8a01..8983310 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/CompactionPlan.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/CompactionPlan.java
@@ -24,6 +24,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import com.google.common.collect.Sets;
@@ -33,8 +34,8 @@ import com.google.common.collect.Sets;
* should simply be deleted, and the optional parameters used to create the resulting output file.
*/
public class CompactionPlan {
- public final List<TabletFile> inputFiles = new ArrayList<>();
- public final List<TabletFile> deleteFiles = new ArrayList<>();
+ public final List<StoredTabletFile> inputFiles = new ArrayList<>();
+ public final List<StoredTabletFile> deleteFiles = new ArrayList<>();
public WriteParameters writeParameters = null;
@Override
@@ -68,9 +69,9 @@ public class CompactionPlan {
* @throws IllegalStateException
* thrown when validation fails.
*/
- public final void validate(Set<TabletFile> allFiles) {
+ public final void validate(Set<StoredTabletFile> allFiles) {
Set<TabletFile> inputSet = new HashSet<>(inputFiles);
- Set<TabletFile> deleteSet = new HashSet<>(deleteFiles);
+ Set<StoredTabletFile> deleteSet = new HashSet<>(deleteFiles);
if (!allFiles.containsAll(inputSet)) {
inputSet.removeAll(allFiles);
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/DefaultCompactionStrategy.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/DefaultCompactionStrategy.java
index 6fef6ec..961f531 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/DefaultCompactionStrategy.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/DefaultCompactionStrategy.java
@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.Map.Entry;
import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
@@ -50,9 +51,9 @@ public class DefaultCompactionStrategy extends CompactionStrategy {
SizeWindow() {}
- SizeWindow(Map<TabletFile,DataFileValue> allFiles) {
+ SizeWindow(Map<StoredTabletFile,DataFileValue> allFiles) {
files = new ArrayList<>();
- for (Entry<TabletFile,DataFileValue> entry : allFiles.entrySet()) {
+ for (Entry<StoredTabletFile,DataFileValue> entry : allFiles.entrySet()) {
files.add(new CompactionFile(entry.getKey(), entry.getValue().getSize()));
}
@@ -117,8 +118,8 @@ public class DefaultCompactionStrategy extends CompactionStrategy {
return (last - first);
}
- public List<TabletFile> getFiles() {
- List<TabletFile> windowFiles = new ArrayList<>(size());
+ public List<StoredTabletFile> getFiles() {
+ List<StoredTabletFile> windowFiles = new ArrayList<>(size());
for (int i = first; i < last; i++) {
windowFiles.add(files.get(i).file);
}
@@ -142,7 +143,7 @@ public class DefaultCompactionStrategy extends CompactionStrategy {
public CompactionPlan getCompactionPlan(MajorCompactionRequest request) {
CompactionPlan result = new CompactionPlan();
- List<TabletFile> toCompact = findMapFilesToCompact(request);
+ List<StoredTabletFile> toCompact = findMapFilesToCompact(request);
if (toCompact == null || toCompact.isEmpty())
return result;
result.inputFiles.addAll(toCompact);
@@ -150,10 +151,10 @@ public class DefaultCompactionStrategy extends CompactionStrategy {
}
private static class CompactionFile {
- public TabletFile file;
+ public StoredTabletFile file;
public long size;
- public CompactionFile(TabletFile file, long size) {
+ public CompactionFile(StoredTabletFile file, long size) {
super();
this.file = file;
this.size = size;
@@ -168,7 +169,7 @@ public class DefaultCompactionStrategy extends CompactionStrategy {
}
}
- private List<TabletFile> findMapFilesToCompact(MajorCompactionRequest request) {
+ private List<StoredTabletFile> findMapFilesToCompact(MajorCompactionRequest request) {
MajorCompactionReason reason = request.getReason();
if (reason == MajorCompactionReason.USER) {
return new ArrayList<>(request.getFiles().keySet());
@@ -195,7 +196,7 @@ public class DefaultCompactionStrategy extends CompactionStrategy {
SizeWindow all = new SizeWindow(request.getFiles());
- List<TabletFile> files = null;
+ List<StoredTabletFile> files = null;
// Within a window of size maxFilesToCompact containing the smallest files check to see if any
// files meet the compaction ratio criteria.
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
index 22fdc91..6a09f84 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java
@@ -40,6 +40,7 @@ import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.dataImpl.TabletIdImpl;
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.spi.cache.BlockCache;
@@ -66,7 +67,7 @@ public class MajorCompactionRequest implements Cloneable {
private final AccumuloConfiguration tableConfig;
private final BlockCache indexCache;
private final BlockCache summaryCache;
- private Map<TabletFile,DataFileValue> files;
+ private Map<StoredTabletFile,DataFileValue> files;
private final ServerContext context;
private final Cache<String,Long> fileLenCache;
@@ -105,7 +106,7 @@ public class MajorCompactionRequest implements Cloneable {
return reason;
}
- public Map<TabletFile,DataFileValue> getFiles() {
+ public Map<StoredTabletFile,DataFileValue> getFiles() {
return files;
}
@@ -149,7 +150,7 @@ public class MajorCompactionRequest implements Cloneable {
* @see TableOperations#addSummarizers(String, SummarizerConfiguration...)
* @see WriterOptions#withSummarizers(SummarizerConfiguration...)
*/
- public List<Summary> getSummaries(Collection<TabletFile> files,
+ public List<Summary> getSummaries(Collection<StoredTabletFile> files,
Predicate<SummarizerConfiguration> summarySelector) {
Objects.requireNonNull(volumeManager,
"Getting summaries is not supported at this time. It's only supported when "
@@ -169,7 +170,7 @@ public class MajorCompactionRequest implements Cloneable {
return sc.getSummaries();
}
- public void setFiles(Map<TabletFile,DataFileValue> update) {
+ public void setFiles(Map<StoredTabletFile,DataFileValue> update) {
this.files = Collections.unmodifiableMap(update);
}
@@ -182,7 +183,7 @@ public class MajorCompactionRequest implements Cloneable {
FileOperations fileFactory = FileOperations.getInstance();
FileSystem ns = volumeManager.getVolumeByPath(tabletFile.getPath()).getFileSystem();
return fileFactory.newReaderBuilder()
- .forFile(tabletFile.getNormalizedPath(), ns, ns.getConf(), context.getCryptoService())
+ .forFile(tabletFile.getPathStr(), ns, ns.getConf(), context.getCryptoService())
.withTableConfiguration(tableConfig).seekToBeginning().build();
}
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategy.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategy.java
index 3ec7fd8..79b4320 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategy.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategy.java
@@ -24,7 +24,7 @@ import java.util.Map;
import java.util.Map.Entry;
import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
-import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.tserver.compaction.strategies.BasicCompactionStrategy;
@@ -42,8 +42,8 @@ public class SizeLimitCompactionStrategy extends DefaultCompactionStrategy {
}
private MajorCompactionRequest filterFiles(MajorCompactionRequest mcr) {
- Map<TabletFile,DataFileValue> filteredFiles = new HashMap<>();
- for (Entry<TabletFile,DataFileValue> entry : mcr.getFiles().entrySet()) {
+ Map<StoredTabletFile,DataFileValue> filteredFiles = new HashMap<>();
+ for (Entry<StoredTabletFile,DataFileValue> entry : mcr.getFiles().entrySet()) {
if (entry.getValue().getSize() <= limit) {
filteredFiles.put(entry.getKey(), entry.getValue());
}
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/strategies/BasicCompactionStrategy.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/strategies/BasicCompactionStrategy.java
index c8639bf..4c5ce35 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/strategies/BasicCompactionStrategy.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/strategies/BasicCompactionStrategy.java
@@ -24,7 +24,7 @@ import java.util.Map;
import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.tserver.compaction.CompactionPlan;
import org.apache.accumulo.tserver.compaction.DefaultCompactionStrategy;
@@ -141,7 +141,7 @@ public class BasicCompactionStrategy extends DefaultCompactionStrategy {
private MajorCompactionRequest filterFiles(MajorCompactionRequest mcr) {
if (filterSize != null) {
- Map<TabletFile,DataFileValue> filteredFiles = new HashMap<>();
+ Map<StoredTabletFile,DataFileValue> filteredFiles = new HashMap<>();
mcr.getFiles().forEach((fr, dfv) -> {
if (dfv.getSize() <= filterSize) {
filteredFiles.put(fr, dfv);
@@ -159,8 +159,8 @@ public class BasicCompactionStrategy extends DefaultCompactionStrategy {
*/
private Long calculateTotalSize(MajorCompactionRequest request, CompactionPlan plan) {
long totalSize = 0;
- Map<TabletFile,DataFileValue> allFiles = request.getFiles();
- for (TabletFile fileRef : plan.inputFiles) {
+ Map<StoredTabletFile,DataFileValue> allFiles = request.getFiles();
+ for (StoredTabletFile fileRef : plan.inputFiles) {
totalSize += allFiles.get(fileRef).getSize();
}
return totalSize;
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/strategies/ConfigurableCompactionStrategy.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/strategies/ConfigurableCompactionStrategy.java
index 2c679c5..6295d4b 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/strategies/ConfigurableCompactionStrategy.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/strategies/ConfigurableCompactionStrategy.java
@@ -36,6 +36,7 @@ import org.apache.accumulo.core.client.summary.Summary;
import org.apache.accumulo.core.compaction.CompactionSettings;
import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
@@ -55,7 +56,7 @@ public class ConfigurableCompactionStrategy extends CompactionStrategy {
// shouldCompact(). See CompactionStrategy javadocs.
void gatherInformation(MajorCompactionRequest request) {}
- abstract boolean shouldCompact(Entry<TabletFile,DataFileValue> file,
+ abstract boolean shouldCompact(Entry<StoredTabletFile,DataFileValue> file,
MajorCompactionRequest request);
}
@@ -87,7 +88,7 @@ public class ConfigurableCompactionStrategy extends CompactionStrategy {
? (Set<SummarizerConfiguration>) configs : new HashSet<>(configs);
okFiles = new HashSet<>();
- for (TabletFile tabletFile : request.getFiles().keySet()) {
+ for (StoredTabletFile tabletFile : request.getFiles().keySet()) {
Map<SummarizerConfiguration,Summary> sMap = new HashMap<>();
Collection<Summary> summaries;
summaries =
@@ -121,7 +122,7 @@ public class ConfigurableCompactionStrategy extends CompactionStrategy {
}
@Override
- public boolean shouldCompact(Entry<TabletFile,DataFileValue> file,
+ public boolean shouldCompact(Entry<StoredTabletFile,DataFileValue> file,
MajorCompactionRequest request) {
if (!gatherCalled) {
@@ -169,7 +170,7 @@ public class ConfigurableCompactionStrategy extends CompactionStrategy {
}
@Override
- public boolean shouldCompact(Entry<TabletFile,DataFileValue> file,
+ public boolean shouldCompact(Entry<StoredTabletFile,DataFileValue> file,
MajorCompactionRequest request) {
if (!gatherCalled) {
@@ -194,7 +195,7 @@ public class ConfigurableCompactionStrategy extends CompactionStrategy {
}
@Override
- public boolean shouldCompact(Entry<TabletFile,DataFileValue> file,
+ public boolean shouldCompact(Entry<StoredTabletFile,DataFileValue> file,
MajorCompactionRequest request) {
return shouldCompact(file.getValue().getSize(), esize);
}
@@ -210,7 +211,7 @@ public class ConfigurableCompactionStrategy extends CompactionStrategy {
}
@Override
- public boolean shouldCompact(Entry<TabletFile,DataFileValue> file,
+ public boolean shouldCompact(Entry<StoredTabletFile,DataFileValue> file,
MajorCompactionRequest request) {
return pattern.matcher(getInput(file.getKey().getPath())).matches();
}
@@ -304,10 +305,10 @@ public class ConfigurableCompactionStrategy extends CompactionStrategy {
}
- private List<TabletFile> getFilesToCompact(MajorCompactionRequest request) {
- List<TabletFile> filesToCompact = new ArrayList<>();
+ private List<StoredTabletFile> getFilesToCompact(MajorCompactionRequest request) {
+ List<StoredTabletFile> filesToCompact = new ArrayList<>();
- for (Entry<TabletFile,DataFileValue> entry : request.getFiles().entrySet()) {
+ for (Entry<StoredTabletFile,DataFileValue> entry : request.getFiles().entrySet()) {
boolean compact = false;
for (Test test : tests) {
if (andTest) {
@@ -341,7 +342,7 @@ public class ConfigurableCompactionStrategy extends CompactionStrategy {
@Override
public CompactionPlan getCompactionPlan(MajorCompactionRequest request) {
- List<TabletFile> filesToCompact = getFilesToCompact(request);
+ List<StoredTabletFile> filesToCompact = getFilesToCompact(request);
if (filesToCompact.size() >= minFiles) {
CompactionPlan plan = new CompactionPlan();
plan.inputFiles.addAll(filesToCompact);
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/strategies/TooManyDeletesCompactionStrategy.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/strategies/TooManyDeletesCompactionStrategy.java
index 566b39a..7e1ffb3 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/strategies/TooManyDeletesCompactionStrategy.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/strategies/TooManyDeletesCompactionStrategy.java
@@ -32,7 +32,7 @@ import org.apache.accumulo.core.client.rfile.RFile.WriterOptions;
import org.apache.accumulo.core.client.summary.SummarizerConfiguration;
import org.apache.accumulo.core.client.summary.Summary;
import org.apache.accumulo.core.client.summary.summarizers.DeletesSummarizer;
-import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.tserver.compaction.CompactionPlan;
import org.apache.accumulo.tserver.compaction.DefaultCompactionStrategy;
@@ -143,7 +143,7 @@ public class TooManyDeletesCompactionStrategy extends DefaultCompactionStrategy
long total = 0;
long deletes = 0;
- for (Entry<TabletFile,DataFileValue> entry : request.getFiles().entrySet()) {
+ for (Entry<StoredTabletFile,DataFileValue> entry : request.getFiles().entrySet()) {
Collection<Summary> summaries =
request.getSummaries(Collections.singleton(entry.getKey()), summarizerPredicate);
if (summaries.size() == 1) {
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactionInfo.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactionInfo.java
index 0be9f6b..cd0e3a2 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactionInfo.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactionInfo.java
@@ -24,9 +24,10 @@ import java.util.List;
import java.util.Map;
import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.clientImpl.Translator;
+import org.apache.accumulo.core.clientImpl.Translators;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.dataImpl.thrift.IterInfo;
-import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction;
import org.apache.accumulo.core.tabletserver.thrift.CompactionReason;
import org.apache.accumulo.core.tabletserver.thrift.CompactionType;
@@ -124,12 +125,9 @@ public class CompactionInfo {
iterSetting.getName()));
iterOptions.put(iterSetting.getName(), iterSetting.getOptions());
}
- List<String> filesToCompact = new ArrayList<>();
- for (TabletFile file : compactor.getFilesToCompact())
- filesToCompact.add(file.getNormalizedPath());
+ List<String> files = Translator.translate(compactor.getFilesToCompact(), Translators.STFT);
return new ActiveCompaction(compactor.extent.toThrift(),
- System.currentTimeMillis() - compactor.getStartTime(), filesToCompact,
- compactor.getOutputFile(), type, reason, localityGroup, entriesRead, entriesWritten, iiList,
- iterOptions);
+ System.currentTimeMillis() - compactor.getStartTime(), files, compactor.getOutputFile(),
+ type, reason, localityGroup, entriesRead, entriesWritten, iiList, iterOptions);
}
}
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
index 71d9c8f..6eee8b0 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Compactor.java
@@ -49,6 +49,7 @@ import org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterato
import org.apache.accumulo.core.iteratorsImpl.system.DeletingIterator;
import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator;
import org.apache.accumulo.core.iteratorsImpl.system.TimeSettingIterator;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.util.LocalityGroupUtil;
@@ -65,7 +66,6 @@ import org.apache.accumulo.tserver.MinorCompactionReason;
import org.apache.accumulo.tserver.TabletIteratorEnvironment;
import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.htrace.Trace;
import org.apache.htrace.TraceScope;
import org.slf4j.Logger;
@@ -90,7 +90,7 @@ public class Compactor implements Callable<CompactionStats> {
RateLimiter getWriteLimiter();
}
- private final Map<TabletFile,DataFileValue> filesToCompact;
+ private final Map<StoredTabletFile,DataFileValue> filesToCompact;
private final InMemoryMap imm;
private final TabletFile outputFile;
private final boolean propogateDeletes;
@@ -147,7 +147,7 @@ public class Compactor implements Callable<CompactionStats> {
return compactions;
}
- public Compactor(ServerContext context, Tablet tablet, Map<TabletFile,DataFileValue> files,
+ public Compactor(ServerContext context, Tablet tablet, Map<StoredTabletFile,DataFileValue> files,
InMemoryMap imm, TabletFile outputFile, boolean propogateDeletes, CompactionEnv env,
List<IteratorSetting> iterators, int reason, AccumuloConfiguration tableConfiguation) {
this.context = context;
@@ -201,8 +201,6 @@ public class Compactor implements Callable<CompactionStats> {
clearStats();
- final Path outputFilePath = outputFile.getPath();
- final String outputFilePathName = outputFile.getNormalizedPath();
String oldThreadName = Thread.currentThread().getName();
String newThreadName = "MajC compacting " + extent + " started "
+ dateFormatter.format(new Date()) + " file: " + outputFile;
@@ -210,9 +208,9 @@ public class Compactor implements Callable<CompactionStats> {
thread = Thread.currentThread();
try {
FileOperations fileFactory = FileOperations.getInstance();
- FileSystem ns = this.fs.getVolumeByPath(outputFilePath).getFileSystem();
+ FileSystem ns = this.fs.getVolumeByPath(outputFile.getPath()).getFileSystem();
mfw = fileFactory.newWriterBuilder()
- .forFile(outputFilePathName, ns, ns.getConf(), context.getCryptoService())
+ .forFile(outputFile.getMetaInsert(), ns, ns.getConf(), context.getCryptoService())
.withTableConfiguration(acuTableConf).withRateLimiter(env.getWriteLimiter()).build();
Map<String,Set<ByteSequence>> lGroups = getLocalityGroups(acuTableConf);
@@ -297,13 +295,13 @@ public class Compactor implements Callable<CompactionStats> {
FileSKVIterator reader;
reader = fileFactory.newReaderBuilder()
- .forFile(mapFile.getMetadataEntry(), fs, fs.getConf(), context.getCryptoService())
+ .forFile(mapFile.getPathStr(), fs, fs.getConf(), context.getCryptoService())
.withTableConfiguration(acuTableConf).withRateLimiter(env.getReadLimiter()).build();
readers.add(reader);
SortedKeyValueIterator<Key,Value> iter = new ProblemReportingIterator(context,
- extent.getTableId(), mapFile.getMetadataEntry(), false, reader);
+ extent.getTableId(), mapFile.getPathStr(), false, reader);
if (filesToCompact.get(mapFile).isTimeSet()) {
iter = new TimeSettingIterator(iter, filesToCompact.get(mapFile).getTime());
@@ -313,8 +311,8 @@ public class Compactor implements Callable<CompactionStats> {
} catch (Throwable e) {
- ProblemReports.getInstance(context).report(new ProblemReport(extent.getTableId(),
- ProblemType.FILE_READ, mapFile.getMetadataEntry(), e));
+ ProblemReports.getInstance(context).report(
+ new ProblemReport(extent.getTableId(), ProblemType.FILE_READ, mapFile.getPathStr(), e));
log.warn("Some problem opening map file {} {}", mapFile, e.getMessage(), e);
// failed to open some map file... close the ones that were opened
@@ -422,7 +420,7 @@ public class Compactor implements Callable<CompactionStats> {
}
}
- Collection<TabletFile> getFilesToCompact() {
+ Collection<StoredTabletFile> getFilesToCompact() {
return filesToCompact.keySet();
}
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
index 6240aca..63293ac 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
@@ -37,6 +37,7 @@ import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.logging.TabletLogger;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.replication.ReplicationConfigurationUtil;
@@ -58,7 +59,7 @@ import org.slf4j.LoggerFactory;
class DatafileManager {
private final Logger log = LoggerFactory.getLogger(DatafileManager.class);
// access to datafilesizes needs to be synchronized: see CompactionRunner#getNumFiles
- private final Map<TabletFile,DataFileValue> datafileSizes =
+ private final Map<StoredTabletFile,DataFileValue> datafileSizes =
Collections.synchronizedMap(new TreeMap<>());
private final Tablet tablet;
private Long maxMergingMinorCompactionFileSize;
@@ -66,8 +67,8 @@ class DatafileManager {
// ensure we only have one reader/writer of our bulk file notes at at time
private final Object bulkFileImportLock = new Object();
- DatafileManager(Tablet tablet, SortedMap<TabletFile,DataFileValue> datafileSizes) {
- for (Entry<TabletFile,DataFileValue> datafiles : datafileSizes.entrySet()) {
+ DatafileManager(Tablet tablet, SortedMap<StoredTabletFile,DataFileValue> datafileSizes) {
+ for (Entry<StoredTabletFile,DataFileValue> datafiles : datafileSizes.entrySet()) {
this.datafileSizes.put(datafiles.getKey(), datafiles.getValue());
}
this.tablet = tablet;
@@ -75,8 +76,8 @@ class DatafileManager {
private TabletFile mergingMinorCompactionFile = null;
private final Set<TabletFile> filesToDeleteAfterScan = new HashSet<>();
- private final Map<Long,Set<TabletFile>> scanFileReservations = new HashMap<>();
- private final MapCounter<TabletFile> fileScanReferenceCounts = new MapCounter<>();
+ private final Map<Long,Set<StoredTabletFile>> scanFileReservations = new HashMap<>();
+ private final MapCounter<StoredTabletFile> fileScanReferenceCounts = new MapCounter<>();
private long nextScanReservationId = 0;
private boolean reservationsBlocked = false;
@@ -99,7 +100,7 @@ class DatafileManager {
}
}
- Set<TabletFile> absFilePaths = new HashSet<>(datafileSizes.keySet());
+ Set<StoredTabletFile> absFilePaths = new HashSet<>(datafileSizes.keySet());
long rid = nextScanReservationId++;
@@ -107,7 +108,7 @@ class DatafileManager {
Map<TabletFile,DataFileValue> ret = new HashMap<>();
- for (TabletFile path : absFilePaths) {
+ for (StoredTabletFile path : absFilePaths) {
fileScanReferenceCounts.increment(path, 1);
ret.put(path, datafileSizes.get(path));
}
@@ -118,16 +119,16 @@ class DatafileManager {
void returnFilesForScan(Long reservationId) {
- final Set<TabletFile> filesToDelete = new HashSet<>();
+ final Set<StoredTabletFile> filesToDelete = new HashSet<>();
synchronized (tablet) {
- Set<TabletFile> absFilePaths = scanFileReservations.remove(reservationId);
+ Set<StoredTabletFile> absFilePaths = scanFileReservations.remove(reservationId);
if (absFilePaths == null)
throw new IllegalArgumentException("Unknown scan reservation id " + reservationId);
boolean notify = false;
- for (TabletFile path : absFilePaths) {
+ for (StoredTabletFile path : absFilePaths) {
long refCount = fileScanReferenceCounts.decrement(path, 1);
if (refCount == 0) {
if (filesToDeleteAfterScan.remove(path))
@@ -148,14 +149,14 @@ class DatafileManager {
}
}
- void removeFilesAfterScan(Set<TabletFile> scanFiles) {
+ void removeFilesAfterScan(Set<StoredTabletFile> scanFiles) {
if (scanFiles.size() == 0)
return;
- Set<TabletFile> filesToDelete = new HashSet<>();
+ Set<StoredTabletFile> filesToDelete = new HashSet<>();
synchronized (tablet) {
- for (TabletFile path : scanFiles) {
+ for (StoredTabletFile path : scanFiles) {
if (fileScanReferenceCounts.get(path) == 0)
filesToDelete.add(path);
else
@@ -170,14 +171,14 @@ class DatafileManager {
}
}
- private TreeSet<TabletFile> waitForScansToFinish(Set<TabletFile> pathsToWaitFor) {
+ private TreeSet<StoredTabletFile> waitForScansToFinish(Set<StoredTabletFile> pathsToWaitFor) {
long maxWait = 10000L;
long startTime = System.currentTimeMillis();
- TreeSet<TabletFile> inUse = new TreeSet<>();
+ TreeSet<StoredTabletFile> inUse = new TreeSet<>();
try (TraceScope waitForScans = Trace.startSpan("waitForScans")) {
synchronized (tablet) {
- for (TabletFile path : pathsToWaitFor) {
+ for (StoredTabletFile path : pathsToWaitFor) {
while (fileScanReferenceCounts.get(path) > 0
&& System.currentTimeMillis() - startTime < maxWait) {
try {
@@ -188,7 +189,7 @@ class DatafileManager {
}
}
- for (TabletFile path : pathsToWaitFor) {
+ for (StoredTabletFile path : pathsToWaitFor) {
if (fileScanReferenceCounts.get(path) > 0)
inUse.add(path);
}
@@ -201,6 +202,8 @@ class DatafileManager {
throws IOException {
String bulkDir = null;
+ // once tablet files are inserted into the metadata they will become StoredTabletFiles
+ Map<StoredTabletFile,DataFileValue> newFiles = new HashMap<>(paths.size());
for (TabletFile tpath : paths.keySet()) {
boolean inTheRightDirectory = false;
@@ -241,17 +244,16 @@ class DatafileManager {
}
}
- tablet.updatePersistedTime(bulkTime, paths, tid);
+ newFiles = tablet.updatePersistedTime(bulkTime, paths, tid);
}
}
synchronized (tablet) {
- for (Entry<TabletFile,DataFileValue> tpath : paths.entrySet()) {
+ for (Entry<StoredTabletFile,DataFileValue> tpath : newFiles.entrySet()) {
if (datafileSizes.containsKey(tpath.getKey())) {
log.error("Adding file that is already in set {}", tpath.getKey());
}
datafileSizes.put(tpath.getKey(), tpath.getValue());
-
}
tablet.getTabletResources().importedMapFiles();
@@ -259,12 +261,12 @@ class DatafileManager {
tablet.computeNumEntries();
}
- for (Entry<TabletFile,DataFileValue> entry : paths.entrySet()) {
+ for (Entry<StoredTabletFile,DataFileValue> entry : newFiles.entrySet()) {
TabletLogger.bulkImported(tablet.getExtent(), entry.getKey());
}
}
- TabletFile reserveMergingMinorCompactionFile() {
+ StoredTabletFile reserveMergingMinorCompactionFile() {
if (mergingMinorCompactionFile != null)
throw new IllegalStateException(
"Tried to reserve merging minor compaction file when already reserved : "
@@ -294,9 +296,9 @@ class DatafileManager {
maxFileSize = maxMergingMinorCompactionFileSize;
}
long min = maxFileSize;
- TabletFile minName = null;
+ StoredTabletFile minName = null;
- for (Entry<TabletFile,DataFileValue> entry : datafileSizes.entrySet()) {
+ for (Entry<StoredTabletFile,DataFileValue> entry : datafileSizes.entrySet()) {
if (entry.getValue().getSize() <= min && !majorCompactingFiles.contains(entry.getKey())) {
min = entry.getValue().getSize();
minName = entry.getKey();
@@ -323,8 +325,8 @@ class DatafileManager {
}
void bringMinorCompactionOnline(TabletFile tmpDatafile, TabletFile newDatafile,
- TabletFile absMergeFile, DataFileValue dfv, CommitSession commitSession, long flushId) {
-
+ StoredTabletFile absMergeFile, DataFileValue dfv, CommitSession commitSession, long flushId) {
+ StoredTabletFile newFile;
// rename before putting in metadata table, so files in metadata table should
// always exist
do {
@@ -359,7 +361,7 @@ class DatafileManager {
// here, but that was incorrect because a scan could start after waiting but before
// memory was updated... assuming the file is always in use by scans leads to
// one unneeded metadata update when it was not actually in use
- Set<TabletFile> filesInUseByScans = Collections.emptySet();
+ Set<StoredTabletFile> filesInUseByScans = Collections.emptySet();
if (absMergeFile != null)
filesInUseByScans = Collections.singleton(absMergeFile);
@@ -367,7 +369,7 @@ class DatafileManager {
// this metadata write does not go up... it goes sideways or to itself
if (absMergeFile != null)
MetadataTableUtil.addDeleteEntries(tablet.getExtent(),
- Collections.singleton(absMergeFile.getMetadataEntry()), tablet.getContext());
+ Collections.singleton(absMergeFile.getMetaUpdateDelete()), tablet.getContext());
Set<String> unusedWalLogs = tablet.beginClearingUnusedLogs();
boolean replicate =
@@ -395,8 +397,8 @@ class DatafileManager {
// following metadata
// write is made
- tablet.updateTabletDataFile(commitSession.getMaxCommittedTime(), newDatafile, absMergeFile,
- dfv, unusedWalLogs, filesInUseByScans, flushId);
+ newFile = tablet.updateTabletDataFile(commitSession.getMaxCommittedTime(), newDatafile,
+ absMergeFile, dfv, unusedWalLogs, filesInUseByScans, flushId);
// Mark that we have data we want to replicate
// This WAL could still be in use by other Tablets *from the same table*, so we can only mark
@@ -437,12 +439,11 @@ class DatafileManager {
synchronized (tablet) {
t1 = System.currentTimeMillis();
- if (datafileSizes.containsKey(newDatafile)) {
- log.error("Adding file that is already in set {}", newDatafile);
- }
-
if (dfv.getNumEntries() > 0) {
- datafileSizes.put(newDatafile, dfv);
+ if (datafileSizes.containsKey(newFile)) {
+ log.error("Adding file that is already in set {}", newFile);
+ }
+ datafileSizes.put(newFile, dfv);
}
if (absMergeFile != null) {
@@ -472,7 +473,7 @@ class DatafileManager {
}
}
- public void reserveMajorCompactingFiles(Collection<TabletFile> files) {
+ public void reserveMajorCompactingFiles(Collection<StoredTabletFile> files) {
if (majorCompactingFiles.size() != 0)
throw new IllegalStateException("Major compacting files not empty " + majorCompactingFiles);
@@ -488,8 +489,9 @@ class DatafileManager {
majorCompactingFiles.clear();
}
- void bringMajorCompactionOnline(Set<TabletFile> oldDatafiles, TabletFile tmpDatafile,
- TabletFile newDatafile, Long compactionId, DataFileValue dfv) throws IOException {
+ StoredTabletFile bringMajorCompactionOnline(Set<StoredTabletFile> oldDatafiles,
+ TabletFile tmpDatafile, TabletFile newDatafile, Long compactionId, DataFileValue dfv)
+ throws IOException {
final KeyExtent extent = tablet.getExtent();
long t1, t2;
@@ -507,14 +509,15 @@ class DatafileManager {
}
TServerInstance lastLocation = null;
+ // calling insert to get the new file before inserting into the metadata
+ StoredTabletFile newFile = newDatafile.insert();
synchronized (tablet) {
-
t1 = System.currentTimeMillis();
tablet.incrementDataSourceDeletions();
// atomically remove old files and add new file
- for (TabletFile oldDatafile : oldDatafiles) {
+ for (StoredTabletFile oldDatafile : oldDatafiles) {
if (!datafileSizes.containsKey(oldDatafile)) {
log.error("file does not exist in set {}", oldDatafile);
}
@@ -522,16 +525,16 @@ class DatafileManager {
majorCompactingFiles.remove(oldDatafile);
}
- if (datafileSizes.containsKey(newDatafile)) {
- log.error("Adding file that is already in set {}", newDatafile);
+ if (datafileSizes.containsKey(newFile)) {
+ log.error("Adding file that is already in set {}", newFile);
}
if (dfv.getNumEntries() > 0) {
- datafileSizes.put(newDatafile, dfv);
+ datafileSizes.put(newFile, dfv);
}
// could be used by a follow on compaction in a multipass compaction
- majorCompactingFiles.add(newDatafile);
+ majorCompactingFiles.add(newFile);
tablet.computeNumEntries();
@@ -541,11 +544,12 @@ class DatafileManager {
t2 = System.currentTimeMillis();
}
- Set<TabletFile> filesInUseByScans = waitForScansToFinish(oldDatafiles);
+ // known consistency issue between minor and major compactions - see ACCUMULO-18
+ Set<StoredTabletFile> filesInUseByScans = waitForScansToFinish(oldDatafiles);
if (filesInUseByScans.size() > 0)
log.debug("Adding scan refs to metadata {} {}", extent, filesInUseByScans);
MasterMetadataUtil.replaceDatafiles(tablet.getContext(), extent, oldDatafiles,
- filesInUseByScans, newDatafile, compactionId, dfv,
+ filesInUseByScans, newFile, compactionId, dfv,
tablet.getTabletServer().getClientAddressString(), lastLocation,
tablet.getTabletServer().getLock());
removeFilesAfterScan(filesInUseByScans);
@@ -554,12 +558,13 @@ class DatafileManager {
log.trace(String.format("MajC finish lock %.2f secs", (t2 - t1) / 1000.0));
}
- TabletLogger.compacted(extent, oldDatafiles, newDatafile);
+ TabletLogger.compacted(extent, oldDatafiles, newFile);
+ return newFile;
}
- public SortedMap<TabletFile,DataFileValue> getDatafileSizes() {
+ public SortedMap<StoredTabletFile,DataFileValue> getDatafileSizes() {
synchronized (tablet) {
- TreeMap<TabletFile,DataFileValue> copy = new TreeMap<>(datafileSizes);
+ TreeMap<StoredTabletFile,DataFileValue> copy = new TreeMap<>(datafileSizes);
return Collections.unmodifiableSortedMap(copy);
}
}
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java
index 2cb6ef0..4750b5f 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java
@@ -20,11 +20,13 @@ package org.apache.accumulo.tserver.tablet;
import java.io.IOException;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.trace.TraceUtil;
import org.apache.accumulo.tserver.MinorCompactionReason;
import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
+import org.apache.hadoop.fs.Path;
import org.apache.htrace.Trace;
import org.apache.htrace.TraceScope;
import org.apache.htrace.impl.ProbabilitySampler;
@@ -39,12 +41,12 @@ class MinorCompactionTask implements Runnable {
private long queued;
private CommitSession commitSession;
private DataFileValue stats;
- private TabletFile mergeFile;
+ private StoredTabletFile mergeFile;
private long flushId;
private MinorCompactionReason mincReason;
private double tracePercent;
- MinorCompactionTask(Tablet tablet, TabletFile mergeFile, CommitSession commitSession,
+ MinorCompactionTask(Tablet tablet, StoredTabletFile mergeFile, CommitSession commitSession,
long flushId, MinorCompactionReason mincReason, double tracePercent) {
this.tablet = tablet;
queued = System.currentTimeMillis();
@@ -62,8 +64,8 @@ class MinorCompactionTask implements Runnable {
ProbabilitySampler sampler = TraceUtil.probabilitySampler(tracePercent);
try {
try (TraceScope minorCompaction = Trace.startSpan("minorCompaction", sampler)) {
- TabletFile newMapfileLocation = tablet.getNextMapFilename(mergeFile == null ? "F" : "M");
- TabletFile tmpFile = new TabletFile(newMapfileLocation.getMetadataEntry() + "_tmp");
+ TabletFile newFile = tablet.getNextMapFilename(mergeFile == null ? "F" : "M");
+ TabletFile tmpFile = new TabletFile(new Path(newFile.getPathStr() + "_tmp"));
try (TraceScope span = Trace.startSpan("waitForCommits")) {
synchronized (tablet) {
commitSession.waitForCommitsToFinish();
@@ -81,7 +83,7 @@ class MinorCompactionTask implements Runnable {
* for the minor compaction
*/
tablet.getTabletServer().minorCompactionStarted(commitSession,
- commitSession.getWALogSeq() + 1, newMapfileLocation.getMetadataEntry());
+ commitSession.getWALogSeq() + 1, newFile.getMetaInsert());
break;
} catch (IOException e) {
log.warn("Failed to write to write ahead log {}", e.getMessage(), e);
@@ -90,7 +92,7 @@ class MinorCompactionTask implements Runnable {
}
try (TraceScope span = Trace.startSpan("compact")) {
this.stats = tablet.minorCompact(tablet.getTabletMemory().getMinCMemTable(), tmpFile,
- newMapfileLocation, mergeFile, queued, commitSession, flushId, mincReason);
+ newFile, mergeFile, queued, commitSession, flushId, mincReason);
}
if (minorCompaction.getSpan() != null) {
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactor.java
index 1167a1f..be90f33 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactor.java
@@ -33,6 +33,7 @@ import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.util.LocalityGroupUtil;
@@ -52,9 +53,10 @@ public class MinorCompactor extends Compactor {
private static final Logger log = LoggerFactory.getLogger(MinorCompactor.class);
- private static final Map<TabletFile,DataFileValue> EMPTY_MAP = Collections.emptyMap();
+ private static final Map<StoredTabletFile,DataFileValue> EMPTY_MAP = Collections.emptyMap();
- private static Map<TabletFile,DataFileValue> toFileMap(TabletFile mergeFile, DataFileValue dfv) {
+ private static Map<StoredTabletFile,DataFileValue> toFileMap(StoredTabletFile mergeFile,
+ DataFileValue dfv) {
if (mergeFile == null)
return EMPTY_MAP;
@@ -64,7 +66,7 @@ public class MinorCompactor extends Compactor {
private final TabletServer tabletServer;
public MinorCompactor(TabletServer tabletServer, Tablet tablet, InMemoryMap imm,
- TabletFile mergeFile, DataFileValue dfv, TabletFile outputFile,
+ StoredTabletFile mergeFile, DataFileValue dfv, TabletFile outputFile,
MinorCompactionReason mincReason, TableConfiguration tableConfig) {
super(tabletServer.getContext(), tablet, toFileMap(mergeFile, dfv), imm, outputFile, true,
new CompactionEnv() {
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 5589fdc..68bde53 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -78,6 +78,7 @@ import org.apache.accumulo.core.logging.TabletLogger;
import org.apache.accumulo.core.master.thrift.BulkImportState;
import org.apache.accumulo.core.master.thrift.TabletLoadState;
import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
@@ -289,8 +290,8 @@ public class Tablet {
TabletFile getNextMapFilename(String prefix) throws IOException {
String extension = FileOperations.getNewFileExtension(tableConfiguration);
- return new TabletFile(chooseTabletDir() + "/" + prefix
- + context.getUniqueNameAllocator().getNextName() + "." + extension);
+ return new TabletFile(new Path(chooseTabletDir() + "/" + prefix
+ + context.getUniqueNameAllocator().getNextName() + "." + extension));
}
private void checkTabletDir(Path path) throws IOException {
@@ -351,7 +352,7 @@ public class Tablet {
}
final List<LogEntry> logEntries = tabletPaths.logEntries;
- final SortedMap<TabletFile,DataFileValue> datafiles = tabletPaths.datafiles;
+ final SortedMap<StoredTabletFile,DataFileValue> datafiles = tabletPaths.datafiles;
constraintChecker = tableConfiguration.newDeriver(conf -> new ConstraintChecker(conf));
@@ -374,7 +375,7 @@ public class Tablet {
try {
Set<String> absPaths = new HashSet<>();
for (TabletFile ref : datafiles.keySet()) {
- absPaths.add(ref.getNormalizedPath());
+ absPaths.add(ref.getPathStr());
}
tabletServer.recover(this.getTabletServer().getFileSystem(), extent, logEntries, absPaths,
@@ -794,7 +795,7 @@ public class Tablet {
}
DataFileValue minorCompact(InMemoryMap memTable, TabletFile tmpDatafile, TabletFile newDatafile,
- TabletFile mergeFile, long queued, CommitSession commitSession, long flushId,
+ StoredTabletFile mergeFile, long queued, CommitSession commitSession, long flushId,
MinorCompactionReason mincReason) {
boolean failed = false;
long start = System.currentTimeMillis();
@@ -854,7 +855,7 @@ public class Tablet {
otherLogs = currentLogs;
currentLogs = new HashSet<>();
- TabletFile mergeFile = null;
+ StoredTabletFile mergeFile = null;
if (mincReason != MinorCompactionReason.RECOVERY) {
mergeFile = getDatafileManager().reserveMergingMinorCompactionFile();
}
@@ -1370,7 +1371,7 @@ public class Tablet {
}
try {
- Pair<List<LogEntry>,SortedMap<TabletFile,DataFileValue>> fileLog =
+ Pair<List<LogEntry>,SortedMap<StoredTabletFile,DataFileValue>> fileLog =
MetadataTableUtil.getFileAndLogEntries(context, extent);
if (fileLog.getFirst().size() != 0) {
@@ -1474,7 +1475,7 @@ public class Tablet {
try {
// we should make .25 below configurable
keys = FileUtil.findMidPoint(context, chooseTabletDir(), extent.getPrevEndRow(),
- extent.getEndRow(), FileUtil.toPathStrings(files), .25);
+ extent.getEndRow(), files, .25);
} catch (IOException e) {
log.error("Failed to find midpoint {}", e.getMessage());
return null;
@@ -1602,16 +1603,16 @@ public class Tablet {
return common;
}
- private Map<TabletFile,Pair<Key,Key>>
- getFirstAndLastKeys(SortedMap<TabletFile,DataFileValue> allFiles) throws IOException {
- final Map<TabletFile,Pair<Key,Key>> result = new HashMap<>();
+ private Map<StoredTabletFile,Pair<Key,Key>>
+ getFirstAndLastKeys(SortedMap<StoredTabletFile,DataFileValue> allFiles) throws IOException {
+ final Map<StoredTabletFile,Pair<Key,Key>> result = new HashMap<>();
final FileOperations fileFactory = FileOperations.getInstance();
final VolumeManager fs = getTabletServer().getFileSystem();
- for (Entry<TabletFile,DataFileValue> entry : allFiles.entrySet()) {
- TabletFile file = entry.getKey();
+ for (Entry<StoredTabletFile,DataFileValue> entry : allFiles.entrySet()) {
+ StoredTabletFile file = entry.getKey();
FileSystem ns = fs.getVolumeByPath(file.getPath()).getFileSystem();
try (FileSKVIterator openReader = fileFactory.newReaderBuilder()
- .forFile(file.getMetadataEntry(), ns, ns.getConf(), context.getCryptoService())
+ .forFile(file.getPathStr(), ns, ns.getConf(), context.getCryptoService())
.withTableConfiguration(this.getTableConfiguration()).seekToBeginning().build()) {
Key first = openReader.getFirstKey();
Key last = openReader.getLastKey();
@@ -1621,15 +1622,15 @@ public class Tablet {
return result;
}
- List<TabletFile> findChopFiles(KeyExtent extent, Map<TabletFile,Pair<Key,Key>> firstAndLastKeys,
- Collection<TabletFile> allFiles) {
- List<TabletFile> result = new ArrayList<>();
+ List<StoredTabletFile> findChopFiles(KeyExtent extent,
+ Map<StoredTabletFile,Pair<Key,Key>> firstAndLastKeys, Collection<StoredTabletFile> allFiles) {
+ List<StoredTabletFile> result = new ArrayList<>();
if (firstAndLastKeys == null) {
result.addAll(allFiles);
return result;
}
- for (TabletFile file : allFiles) {
+ for (StoredTabletFile file : allFiles) {
Pair<Key,Key> pair = firstAndLastKeys.get(file);
if (pair == null) {
// file was created or imported after we obtained the first and last keys... there
@@ -1671,7 +1672,7 @@ public class Tablet {
Pair<Long,UserCompactionConfig> compactionId = null;
CompactionStrategy strategy = null;
- Map<TabletFile,Pair<Key,Key>> firstAndLastKeys = null;
+ Map<StoredTabletFile,Pair<Key,Key>> firstAndLastKeys = null;
if (reason == MajorCompactionReason.USER) {
try {
@@ -1703,7 +1704,7 @@ public class Tablet {
strategy.gatherInformation(request);
}
- Map<TabletFile,DataFileValue> filesToCompact = null;
+ Map<StoredTabletFile,DataFileValue> filesToCompact = null;
int maxFilesToCompact = tableConfiguration.getCount(Property.TSERV_MAJC_THREAD_MAXOPEN);
@@ -1728,8 +1729,8 @@ public class Tablet {
majorCompactionState = CompactionState.IN_PROGRESS;
notifyAll();
- SortedMap<TabletFile,DataFileValue> allFiles = getDatafileManager().getDatafileSizes();
- List<TabletFile> inputFiles = new ArrayList<>();
+ SortedMap<StoredTabletFile,DataFileValue> allFiles = getDatafileManager().getDatafileSizes();
+ List<StoredTabletFile> inputFiles = new ArrayList<>();
if (reason == MajorCompactionReason.CHOP) {
// enforce rules: files with keys outside our range need to be compacted
inputFiles.addAll(findChopFiles(extent, firstAndLastKeys, allFiles.keySet()));
@@ -1761,7 +1762,7 @@ public class Tablet {
} else {
// If no original files will exist at the end of the compaction, we do not have to propagate
// deletes
- Set<TabletFile> droppedFiles = new HashSet<>();
+ Set<StoredTabletFile> droppedFiles = new HashSet<>();
droppedFiles.addAll(inputFiles);
if (plan != null) {
droppedFiles.addAll(plan.deleteFiles);
@@ -1832,11 +1833,11 @@ public class Tablet {
numToCompact = filesToCompact.size() - maxFilesToCompact + 1;
}
- Set<TabletFile> smallestFiles = removeSmallest(filesToCompact, numToCompact);
+ Set<StoredTabletFile> smallestFiles = removeSmallest(filesToCompact, numToCompact);
- TabletFile fileName =
+ TabletFile newFile =
getNextMapFilename((filesToCompact.size() == 0 && !propogateDeletes) ? "A" : "C");
- TabletFile compactTmpName = new TabletFile(fileName.getMetadataEntry() + "_tmp");
+ TabletFile compactTmpName = new TabletFile(new Path(newFile.getMetaInsert() + "_tmp"));
AccumuloConfiguration tableConf = createCompactionConfiguration(tableConfiguration, plan);
@@ -1864,7 +1865,7 @@ public class Tablet {
};
- HashMap<TabletFile,DataFileValue> copy =
+ HashMap<StoredTabletFile,DataFileValue> copy =
new HashMap<>(getDatafileManager().getDatafileSizes());
if (!copy.keySet().containsAll(smallestFiles)) {
throw new IllegalStateException("Cannot find data file values for " + smallestFiles
@@ -1894,14 +1895,16 @@ public class Tablet {
if (lastBatch && plan != null && plan.deleteFiles != null) {
smallestFiles.addAll(plan.deleteFiles);
}
- getDatafileManager().bringMajorCompactionOnline(smallestFiles, compactTmpName, fileName,
- filesToCompact.size() == 0 && compactionId != null ? compactionId.getFirst() : null,
- new DataFileValue(mcs.getFileSize(), mcs.getEntriesWritten()));
+ StoredTabletFile newTabletFile =
+ getDatafileManager().bringMajorCompactionOnline(smallestFiles, compactTmpName,
+ newFile, filesToCompact.size() == 0 && compactionId != null
+ ? compactionId.getFirst() : null,
+ new DataFileValue(mcs.getFileSize(), mcs.getEntriesWritten()));
// when major compaction produces a file w/ zero entries, it will be deleted... do not
// want to add the deleted file
if (filesToCompact.size() > 0 && mcs.getEntriesWritten() > 0) {
- filesToCompact.put(fileName,
+ filesToCompact.put(newTabletFile,
new DataFileValue(mcs.getFileSize(), mcs.getEntriesWritten()));
}
}
@@ -1940,18 +1943,18 @@ public class Tablet {
return result;
}
- private Set<TabletFile> removeSmallest(Map<TabletFile,DataFileValue> filesToCompact,
+ private Set<StoredTabletFile> removeSmallest(Map<StoredTabletFile,DataFileValue> filesToCompact,
int maxFilesToCompact) {
// ensure this method works properly when multiple files have the same size
// short-circuit; also handles zero files case
if (filesToCompact.size() <= maxFilesToCompact) {
- Set<TabletFile> smallestFiles = new HashSet<>(filesToCompact.keySet());
+ Set<StoredTabletFile> smallestFiles = new HashSet<>(filesToCompact.keySet());
filesToCompact.clear();
return smallestFiles;
}
- PriorityQueue<Pair<TabletFile,Long>> fileHeap =
+ PriorityQueue<Pair<StoredTabletFile,Long>> fileHeap =
new PriorityQueue<>(filesToCompact.size(), (o1, o2) -> {
if (o1.getSecond().equals(o2.getSecond())) {
return o1.getFirst().compareTo(o2.getFirst());
@@ -1962,13 +1965,13 @@ public class Tablet {
return 1;
});
- for (Entry<TabletFile,DataFileValue> entry : filesToCompact.entrySet()) {
+ for (Entry<StoredTabletFile,DataFileValue> entry : filesToCompact.entrySet()) {
fileHeap.add(new Pair<>(entry.getKey(), entry.getValue().getSize()));
}
- Set<TabletFile> smallestFiles = new HashSet<>();
+ Set<StoredTabletFile> smallestFiles = new HashSet<>();
while (smallestFiles.size() < maxFilesToCompact && fileHeap.size() > 0) {
- Pair<TabletFile,Long> pair = fileHeap.remove();
+ Pair<StoredTabletFile,Long> pair = fileHeap.remove();
filesToCompact.remove(pair.getFirst());
smallestFiles.add(pair.getFirst());
}
@@ -2150,10 +2153,8 @@ public class Tablet {
splitPoint = findSplitRow(getDatafileManager().getFiles());
} else {
Text tsp = new Text(sp);
- splitPoint = new SplitRowSpec(
- FileUtil.estimatePercentageLTE(context, chooseTabletDir(), extent.getPrevEndRow(),
- extent.getEndRow(), FileUtil.toPathStrings(getDatafileManager().getFiles()), tsp),
- tsp);
+ splitPoint = new SplitRowSpec(FileUtil.estimatePercentageLTE(context, chooseTabletDir(),
+ extent.getPrevEndRow(), extent.getEndRow(), getDatafileManager().getFiles(), tsp), tsp);
}
if (splitPoint == null || splitPoint.row == null) {
@@ -2174,9 +2175,9 @@ public class Tablet {
String lowDirectoryName = createTabletDirectoryName(context, midRow);
// write new tablet information to MetadataTable
- SortedMap<TabletFile,DataFileValue> lowDatafileSizes = new TreeMap<>();
- SortedMap<TabletFile,DataFileValue> highDatafileSizes = new TreeMap<>();
- List<TabletFile> highDatafilesToRemove = new ArrayList<>();
+ SortedMap<StoredTabletFile,DataFileValue> lowDatafileSizes = new TreeMap<>();
+ SortedMap<StoredTabletFile,DataFileValue> highDatafileSizes = new TreeMap<>();
+ List<StoredTabletFile> highDatafilesToRemove = new ArrayList<>();
MetadataTableUtil.splitDatafiles(midRow, splitRatio, firstAndLastRows,
getDatafileManager().getDatafileSizes(), lowDatafileSizes, highDatafileSizes,
@@ -2211,7 +2212,7 @@ public class Tablet {
}
}
- public SortedMap<TabletFile,DataFileValue> getDatafiles() {
+ public SortedMap<StoredTabletFile,DataFileValue> getDatafiles() {
return getDatafileManager().getDatafileSizes();
}
@@ -2263,7 +2264,7 @@ public class Tablet {
for (Entry<TabletFile,MapFileInfo> entry : fileMap.entrySet()) {
entries.put(entry.getKey(), new DataFileValue(entry.getValue().estimatedSize, 0L));
- files.add(entry.getKey().getNormalizedPath());
+ files.add(entry.getKey().getMetaInsert());
}
// Clients timeout and will think that this operation failed.
@@ -2661,31 +2662,32 @@ public class Tablet {
return tabletServer;
}
- public void updatePersistedTime(long bulkTime, Map<TabletFile,DataFileValue> paths, long tid) {
+ public Map<StoredTabletFile,DataFileValue> updatePersistedTime(long bulkTime,
+ Map<TabletFile,DataFileValue> paths, long tid) {
synchronized (timeLock) {
if (bulkTime > persistedTime) {
persistedTime = bulkTime;
}
- MetadataTableUtil.updateTabletDataFile(tid, extent, paths,
+ return MetadataTableUtil.updateTabletDataFile(tid, extent, paths,
tabletTime.getMetadataTime(persistedTime), getTabletServer().getContext(),
getTabletServer().getLock());
}
}
- public void updateTabletDataFile(long maxCommittedTime, TabletFile newDatafile,
- TabletFile absMergeFile, DataFileValue dfv, Set<String> unusedWalLogs,
- Set<TabletFile> filesInUseByScans, long flushId) {
+ public StoredTabletFile updateTabletDataFile(long maxCommittedTime, TabletFile newDatafile,
+ StoredTabletFile absMergeFile, DataFileValue dfv, Set<String> unusedWalLogs,
+ Set<StoredTabletFile> filesInUseByScans, long flushId) {
synchronized (timeLock) {
if (maxCommittedTime > persistedTime) {
persistedTime = maxCommittedTime;
}
- MasterMetadataUtil.updateTabletDataFile(getTabletServer().getContext(), extent, newDatafile,
- absMergeFile, dfv, tabletTime.getMetadataTime(persistedTime), filesInUseByScans,
- tabletServer.getClientAddressString(), tabletServer.getLock(), unusedWalLogs,
- lastLocation, flushId);
+ return MasterMetadataUtil.updateTabletDataFile(getTabletServer().getContext(), extent,
+ newDatafile, absMergeFile, dfv, tabletTime.getMetadataTime(persistedTime),
+ filesInUseByScans, tabletServer.getClientAddressString(), tabletServer.getLock(),
+ unusedWalLogs, lastLocation, flushId);
}
}
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java
index 8ff52d3..fd3ca0a 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java
@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.metadata.schema.MetadataTime;
@@ -38,9 +39,9 @@ import org.apache.accumulo.server.master.state.TServerInstance;
*/
public class TabletData {
private MetadataTime time = null;
- private SortedMap<TabletFile,DataFileValue> dataFiles = new TreeMap<>();
+ private SortedMap<StoredTabletFile,DataFileValue> dataFiles = new TreeMap<>();
private List<LogEntry> logEntries = new ArrayList<>();
- private HashSet<TabletFile> scanFiles = new HashSet<>();
+ private HashSet<StoredTabletFile> scanFiles = new HashSet<>();
private long flushID = -1;
private long compactID = -1;
private TServerInstance lastLocation = null;
@@ -69,7 +70,7 @@ public class TabletData {
}
// Data pulled from an existing tablet to make a split
- public TabletData(String dirName, SortedMap<TabletFile,DataFileValue> highDatafileSizes,
+ public TabletData(String dirName, SortedMap<StoredTabletFile,DataFileValue> highDatafileSizes,
MetadataTime time, long lastFlushID, long lastCompactID, TServerInstance lastLocation,
Map<Long,List<TabletFile>> bulkIngestedFiles) {
this.directoryName = dirName;
@@ -86,7 +87,7 @@ public class TabletData {
return time;
}
- public SortedMap<TabletFile,DataFileValue> getDataFiles() {
+ public SortedMap<StoredTabletFile,DataFileValue> getDataFiles() {
return dataFiles;
}
@@ -94,7 +95,7 @@ public class TabletData {
return logEntries;
}
- public HashSet<TabletFile> getScanFiles() {
+ public HashSet<StoredTabletFile> getScanFiles() {
return scanFiles;
}
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/CompactionPlanTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/CompactionPlanTest.java
index fa1ca42..09619b2 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/CompactionPlanTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/CompactionPlanTest.java
@@ -20,7 +20,7 @@ package org.apache.accumulo.tserver.compaction;
import java.util.Set;
-import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -34,15 +34,15 @@ public class CompactionPlanTest {
public void testOverlappingInputAndDelete() {
CompactionPlan cp1 = new CompactionPlan();
- TabletFile fr1 = new TabletFile("hdfs://nn1/accumulo/tables/1/t-1/1.rf");
- TabletFile fr2 = new TabletFile("hdfs://nn1/accumulo/tables/1/t-1/2.rf");
+ StoredTabletFile fr1 = new StoredTabletFile("hdfs://nn1/accumulo/tables/1/t-1/1.rf");
+ StoredTabletFile fr2 = new StoredTabletFile("hdfs://nn1/accumulo/tables/1/t-1/2.rf");
cp1.inputFiles.add(fr1);
cp1.deleteFiles.add(fr1);
cp1.deleteFiles.add(fr2);
- Set<TabletFile> allFiles = Set.of(fr1, fr2);
+ Set<StoredTabletFile> allFiles = Set.of(fr1, fr2);
exception.expect(IllegalStateException.class);
cp1.validate(allFiles);
@@ -52,15 +52,15 @@ public class CompactionPlanTest {
public void testInputNotInAllFiles() {
CompactionPlan cp1 = new CompactionPlan();
- TabletFile fr1 = new TabletFile("hdfs://nn1/accumulo/tables/1/t-1/1.rf");
- TabletFile fr2 = new TabletFile("hdfs://nn1/accumulo/tables/1/t-1/2.rf");
- TabletFile fr3 = new TabletFile("hdfs://nn1/accumulo/tables/1/t-2/3.rf");
+ StoredTabletFile fr1 = new StoredTabletFile("hdfs://nn1/accumulo/tables/1/t-1/1.rf");
+ StoredTabletFile fr2 = new StoredTabletFile("hdfs://nn1/accumulo/tables/1/t-1/2.rf");
+ StoredTabletFile fr3 = new StoredTabletFile("hdfs://nn1/accumulo/tables/1/t-2/3.rf");
cp1.inputFiles.add(fr1);
cp1.inputFiles.add(fr2);
cp1.inputFiles.add(fr3);
- Set<TabletFile> allFiles = Set.of(fr1, fr2);
+ Set<StoredTabletFile> allFiles = Set.of(fr1, fr2);
exception.expect(IllegalStateException.class);
cp1.validate(allFiles);
@@ -70,15 +70,15 @@ public class CompactionPlanTest {
public void testDeleteNotInAllFiles() {
CompactionPlan cp1 = new CompactionPlan();
- TabletFile fr1 = new TabletFile("hdfs://nn1/accumulo/tables/1/t-1/1.rf");
- TabletFile fr2 = new TabletFile("hdfs://nn1/accumulo/tables/1/t-1/2.rf");
- TabletFile fr3 = new TabletFile("hdfs://nn1/accumulo/tables/1/t-2/3.rf");
+ StoredTabletFile fr1 = new StoredTabletFile("hdfs://nn1/accumulo/tables/1/t-1/1.rf");
+ StoredTabletFile fr2 = new StoredTabletFile("hdfs://nn1/accumulo/tables/1/t-1/2.rf");
+ StoredTabletFile fr3 = new StoredTabletFile("hdfs://nn1/accumulo/tables/1/t-2/3.rf");
cp1.deleteFiles.add(fr1);
cp1.deleteFiles.add(fr2);
cp1.deleteFiles.add(fr3);
- Set<TabletFile> allFiles = Set.of(fr1, fr2);
+ Set<StoredTabletFile> allFiles = Set.of(fr1, fr2);
exception.expect(IllegalStateException.class);
cp1.validate(allFiles);
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/DefaultCompactionStrategyTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/DefaultCompactionStrategyTest.java
index 8b7a4b7..9114981 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/DefaultCompactionStrategyTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/DefaultCompactionStrategyTest.java
@@ -50,6 +50,7 @@ import org.apache.accumulo.core.file.FileSKVIterator;
import org.apache.accumulo.core.file.blockfile.impl.CacheProvider;
import org.apache.accumulo.core.iterators.IteratorEnvironment;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
@@ -178,13 +179,13 @@ public class DefaultCompactionStrategyTest {
}
TestCompactionRequest(KeyExtent extent, MajorCompactionReason reason,
- Map<TabletFile,DataFileValue> files) {
+ Map<StoredTabletFile,DataFileValue> files) {
super(extent, reason, dfault, getServerContext());
setFiles(files);
}
TestCompactionRequest(KeyExtent extent, MajorCompactionReason reason,
- Map<TabletFile,DataFileValue> files, AccumuloConfiguration config) {
+ Map<StoredTabletFile,DataFileValue> files, AccumuloConfiguration config) {
super(extent, reason, config, getServerContext());
setFiles(files);
}
@@ -202,10 +203,10 @@ public class DefaultCompactionStrategyTest {
}
- static Map<TabletFile,DataFileValue> createFileMap(Object... objs) {
- Map<TabletFile,DataFileValue> files = new HashMap<>();
+ static Map<StoredTabletFile,DataFileValue> createFileMap(Object... objs) {
+ Map<StoredTabletFile,DataFileValue> files = new HashMap<>();
for (int i = 0; i < objs.length; i += 2) {
- files.put(new TabletFile("hdfs://nn1/accumulo/tables/5/t-0001/" + objs[i]),
+ files.put(new StoredTabletFile("hdfs://nn1/accumulo/tables/5/t-0001/" + objs[i]),
new DataFileValue(((Number) objs[i + 1]).longValue(), 0));
}
return files;
@@ -224,10 +225,10 @@ public class DefaultCompactionStrategyTest {
return asSet(Arrays.asList(strings));
}
- private static Set<String> asStringSet(Collection<TabletFile> refs) {
+ private static Set<String> asStringSet(Collection<StoredTabletFile> refs) {
HashSet<String> result = new HashSet<>();
for (TabletFile ref : refs) {
- result.add(ref.getNormalizedPath());
+ result.add(ref.getPathStr());
}
return result;
}
@@ -330,7 +331,7 @@ public class DefaultCompactionStrategyTest {
private ConfigurationCopy config;
int nextFile = 0;
- Map<TabletFile,DataFileValue> files = new HashMap<>();
+ Map<StoredTabletFile,DataFileValue> files = new HashMap<>();
long totalRead = 0;
long added = 0;
@@ -348,7 +349,7 @@ public class DefaultCompactionStrategyTest {
"hdfs://nn1/accumulo/tables/5/t-0001/I" + String.format("%06d", nextFile) + ".rf";
nextFile++;
- files.put(new TabletFile(name), new DataFileValue(size, entries));
+ files.put(new StoredTabletFile(name), new DataFileValue(size, entries));
added += size;
}
}
@@ -366,7 +367,7 @@ public class DefaultCompactionStrategyTest {
long totalSize = 0;
long totalEntries = 0;
- for (TabletFile fr : plan.inputFiles) {
+ for (StoredTabletFile fr : plan.inputFiles) {
DataFileValue dfv = files.remove(fr);
totalSize += dfv.getSize();
@@ -379,7 +380,7 @@ public class DefaultCompactionStrategyTest {
"hdfs://nn1/accumulo/tables/5/t-0001/C" + String.format("%06d", nextFile) + ".rf";
nextFile++;
- files.put(new TabletFile(name), new DataFileValue(totalSize, totalEntries));
+ files.put(new StoredTabletFile(name), new DataFileValue(totalSize, totalEntries));
return totalSize;
@@ -397,12 +398,12 @@ public class DefaultCompactionStrategyTest {
}
void print() {
- List<Entry<TabletFile,DataFileValue>> entries = new ArrayList<>(files.entrySet());
+ List<Entry<StoredTabletFile,DataFileValue>> entries = new ArrayList<>(files.entrySet());
Collections.sort(entries,
(e1, e2) -> Long.compare(e2.getValue().getSize(), e1.getValue().getSize()));
- for (Entry<TabletFile,DataFileValue> entry : entries) {
+ for (Entry<StoredTabletFile,DataFileValue> entry : entries) {
System.out.printf("%s %,d %,d\n", entry.getKey().getFileName(), entry.getValue().getSize(),
entry.getValue().getNumEntries());
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategyTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategyTest.java
index fc74642..258c078 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategyTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategyTest.java
@@ -32,17 +32,17 @@ import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.junit.Test;
public class SizeLimitCompactionStrategyTest {
- private static Map<TabletFile,DataFileValue> nfl(String... sa) {
+ private static Map<StoredTabletFile,DataFileValue> nfl(String... sa) {
- HashMap<TabletFile,DataFileValue> ret = new HashMap<>();
+ HashMap<StoredTabletFile,DataFileValue> ret = new HashMap<>();
for (int i = 0; i < sa.length; i += 2) {
- ret.put(new TabletFile("hdfs://nn1/accumulo/tables/5/t-0001/" + sa[i]),
+ ret.put(new StoredTabletFile("hdfs://nn1/accumulo/tables/5/t-0001/" + sa[i]),
new DataFileValue(ConfigurationTypeHelper.getFixedMemoryAsBytes(sa[i + 1]), 1));
}
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/SizeWindowTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/SizeWindowTest.java
index ab380ca..4b9e0fb 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/SizeWindowTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/SizeWindowTest.java
@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.stream.Collectors;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.tserver.compaction.DefaultCompactionStrategy.SizeWindow;
@@ -34,10 +35,10 @@ import org.junit.Test;
public class SizeWindowTest {
static class TestSizeWindow extends SizeWindow {
- private static Map<TabletFile,DataFileValue> convert(Map<String,Integer> testData) {
- Map<TabletFile,DataFileValue> files = new HashMap<>();
+ private static Map<StoredTabletFile,DataFileValue> convert(Map<String,Integer> testData) {
+ Map<StoredTabletFile,DataFileValue> files = new HashMap<>();
testData.forEach((k, v) -> {
- files.put(new TabletFile("hdfs://nn1/accumulo/tables/5/t-0001/" + k),
+ files.put(new StoredTabletFile("hdfs://nn1/accumulo/tables/5/t-0001/" + k),
new DataFileValue(v, 0));
});
return files;
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/strategies/BasicCompactionStrategyTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/strategies/BasicCompactionStrategyTest.java
index 66d6ee0..b8b4625 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/strategies/BasicCompactionStrategyTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/strategies/BasicCompactionStrategyTest.java
@@ -35,7 +35,7 @@ import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.tserver.InMemoryMapTest;
import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
@@ -54,11 +54,11 @@ public class BasicCompactionStrategyTest {
private AccumuloConfiguration conf = null;
private HashMap<String,String> opts = new HashMap<>();
- private Map<TabletFile,DataFileValue> createFileMap(String... sa) {
+ private Map<StoredTabletFile,DataFileValue> createFileMap(String... sa) {
- HashMap<TabletFile,DataFileValue> ret = new HashMap<>();
+ HashMap<StoredTabletFile,DataFileValue> ret = new HashMap<>();
for (int i = 0; i < sa.length; i += 2) {
- ret.put(new TabletFile("hdfs://nn1/accumulo/tables/5/t-0001/" + sa[i]),
+ ret.put(new StoredTabletFile("hdfs://nn1/accumulo/tables/5/t-0001/" + sa[i]),
new DataFileValue(ConfigurationTypeHelper.getFixedMemoryAsBytes(sa[i + 1]), 1));
}
@@ -79,14 +79,14 @@ public class BasicCompactionStrategyTest {
KeyExtent ke = new KeyExtent(TableId.of("0"), null, null);
mcr = new MajorCompactionRequest(ke, MajorCompactionReason.NORMAL, conf,
InMemoryMapTest.getServerContext());
- Map<TabletFile,DataFileValue> fileMap = createFileMap("f1", "10M", "f2", "10M", "f3", "10M",
- "f4", "10M", "f5", "100M", "f6", "100M", "f7", "100M", "f8", "100M");
+ Map<StoredTabletFile,DataFileValue> fileMap = createFileMap("f1", "10M", "f2", "10M", "f3",
+ "10M", "f4", "10M", "f5", "100M", "f6", "100M", "f7", "100M", "f8", "100M");
mcr.setFiles(fileMap);
assertTrue(ttcs.shouldCompact(mcr));
assertEquals(8, mcr.getFiles().size());
- List<TabletFile> filesToCompact = ttcs.getCompactionPlan(mcr).inputFiles;
+ List<StoredTabletFile> filesToCompact = ttcs.getCompactionPlan(mcr).inputFiles;
assertEquals(fileMap.keySet(), new HashSet<>(filesToCompact));
assertEquals(8, filesToCompact.size());
assertNull(ttcs.getCompactionPlan(mcr).writeParameters);
@@ -98,14 +98,14 @@ public class BasicCompactionStrategyTest {
conf = DefaultConfiguration.getInstance();
KeyExtent ke = new KeyExtent(TableId.of("0"), null, null);
mcr = new MajorCompactionRequest(ke, MajorCompactionReason.NORMAL, conf, getServerContext());
- Map<TabletFile,DataFileValue> fileMap =
+ Map<StoredTabletFile,DataFileValue> fileMap =
createFileMap("f1", "2G", "f2", "2G", "f3", "2G", "f4", "2G");
mcr.setFiles(fileMap);
assertTrue(ttcs.shouldCompact(mcr));
assertEquals(4, mcr.getFiles().size());
- List<TabletFile> filesToCompact = ttcs.getCompactionPlan(mcr).inputFiles;
+ List<StoredTabletFile> filesToCompact = ttcs.getCompactionPlan(mcr).inputFiles;
assertEquals(fileMap.keySet(), new HashSet<>(filesToCompact));
assertEquals(4, filesToCompact.size());
assertEquals(largeCompressionType,
@@ -140,16 +140,16 @@ public class BasicCompactionStrategyTest {
conf = DefaultConfiguration.getInstance();
KeyExtent ke = new KeyExtent(TableId.of("0"), null, null);
mcr = new MajorCompactionRequest(ke, MajorCompactionReason.NORMAL, conf, getServerContext());
- Map<TabletFile,DataFileValue> fileMap = createFileMap("f1", "1G", "f2", "10M", "f3", "10M",
- "f4", "10M", "f5", "10M", "f6", "10M", "f7", "10M");
- Map<TabletFile,DataFileValue> filesToCompactMap =
+ Map<StoredTabletFile,DataFileValue> fileMap = createFileMap("f1", "1G", "f2", "10M", "f3",
+ "10M", "f4", "10M", "f5", "10M", "f6", "10M", "f7", "10M");
+ Map<StoredTabletFile,DataFileValue> filesToCompactMap =
createFileMap("f2", "10M", "f3", "10M", "f4", "10M", "f5", "10M", "f6", "10M", "f7", "10M");
mcr.setFiles(fileMap);
assertTrue(ttcs.shouldCompact(mcr));
assertEquals(7, mcr.getFiles().size());
- List<TabletFile> filesToCompact = ttcs.getCompactionPlan(mcr).inputFiles;
+ List<StoredTabletFile> filesToCompact = ttcs.getCompactionPlan(mcr).inputFiles;
assertEquals(filesToCompactMap.keySet(), new HashSet<>(filesToCompact));
assertEquals(6, filesToCompact.size());
assertNull(ttcs.getCompactionPlan(mcr).writeParameters);
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/strategies/ConfigurableCompactionStrategyTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/strategies/ConfigurableCompactionStrategyTest.java
index 7501a73..bbfcec1 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/strategies/ConfigurableCompactionStrategyTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/strategies/ConfigurableCompactionStrategyTest.java
@@ -29,7 +29,7 @@ import org.apache.accumulo.core.compaction.CompactionSettings;
import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.tserver.compaction.CompactionPlan;
import org.apache.accumulo.tserver.compaction.MajorCompactionReason;
@@ -46,8 +46,8 @@ public class ConfigurableCompactionStrategyTest {
new MajorCompactionRequest(new KeyExtent(TableId.of("1"), null, null),
MajorCompactionReason.USER, null, getServerContext());
- Map<TabletFile,DataFileValue> files = new HashMap<>();
- files.put(new TabletFile("hdfs://nn1/accumulo/tables/1/t-009/F00001.rf"),
+ Map<StoredTabletFile,DataFileValue> files = new HashMap<>();
+ files.put(new StoredTabletFile("hdfs://nn1/accumulo/tables/1/t-009/F00001.rf"),
new DataFileValue(50000, 400));
mcr.setFiles(files);
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
index ffe2248..4e7b0be 100644
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java
@@ -27,6 +27,7 @@ import java.util.TreeMap;
import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.server.conf.TableConfiguration;
@@ -42,10 +43,10 @@ public class DatafileManagerTest {
private KeyExtent extent;
private TableConfiguration tableConf;
- private SortedMap<TabletFile,DataFileValue> createFileMap(String... sa) {
- SortedMap<TabletFile,DataFileValue> ret = new TreeMap<>();
+ private SortedMap<StoredTabletFile,DataFileValue> createFileMap(String... sa) {
+ SortedMap<StoredTabletFile,DataFileValue> ret = new TreeMap<>();
for (int i = 0; i < sa.length; i += 2) {
- ret.put(new TabletFile("hdfs://nn1/accumulo/tables/5/t-0001/" + sa[i]),
+ ret.put(new StoredTabletFile("hdfs://nn1/accumulo/tables/5/t-0001/" + sa[i]),
new DataFileValue(ConfigurationTypeHelper.getFixedMemoryAsBytes(sa[i + 1]), 1));
}
return ret;
@@ -74,7 +75,7 @@ public class DatafileManagerTest {
.andReturn(maxMergeFileSize);
EasyMock.replay(tablet, tableConf);
- SortedMap<TabletFile,DataFileValue> testFiles = createFileMap("largefile", "10M", "file2",
+ SortedMap<StoredTabletFile,DataFileValue> testFiles = createFileMap("largefile", "10M", "file2",
"100M", "file3", "100M", "file4", "100M", "file5", "100M");
DatafileManager dfm = new DatafileManager(tablet, testFiles);
@@ -92,7 +93,7 @@ public class DatafileManagerTest {
public void testReserveMergingMinorCompactionFile_MaxFilesNotReached() {
EasyMock.replay(tablet, tableConf);
- SortedMap<TabletFile,DataFileValue> testFiles =
+ SortedMap<StoredTabletFile,DataFileValue> testFiles =
createFileMap("smallfile", "100B", "file2", "100M", "file3", "100M", "file4", "100M");
DatafileManager dfm = new DatafileManager(tablet, testFiles);
@@ -114,8 +115,8 @@ public class DatafileManagerTest {
.andReturn(maxMergeFileSize);
EasyMock.replay(tablet, tableConf);
- SortedMap<TabletFile,DataFileValue> testFiles = createFileMap("smallfile", "100B", "file2",
- "100M", "file3", "100M", "file4", "100M", "file5", "100M");
+ SortedMap<StoredTabletFile,DataFileValue> testFiles = createFileMap("smallfile", "100B",
+ "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
DatafileManager dfm = new DatafileManager(tablet, testFiles);
TabletFile mergeFile = dfm.reserveMergingMinorCompactionFile();
@@ -136,8 +137,8 @@ public class DatafileManagerTest {
.andReturn(maxMergeFileSize);
EasyMock.replay(tablet, tableConf);
- SortedMap<TabletFile,DataFileValue> testFiles = createFileMap("smallishfile", "10M", "file2",
- "100M", "file3", "100M", "file4", "100M", "file5", "100M");
+ SortedMap<StoredTabletFile,DataFileValue> testFiles = createFileMap("smallishfile", "10M",
+ "file2", "100M", "file3", "100M", "file4", "100M", "file5", "100M");
DatafileManager dfm = new DatafileManager(tablet, testFiles);
TabletFile mergeFile = dfm.reserveMergingMinorCompactionFile();
diff --git a/test/src/main/java/org/apache/accumulo/test/SizeCompactionStrategy.java b/test/src/main/java/org/apache/accumulo/test/SizeCompactionStrategy.java
index 65665dd..5effa79 100644
--- a/test/src/main/java/org/apache/accumulo/test/SizeCompactionStrategy.java
+++ b/test/src/main/java/org/apache/accumulo/test/SizeCompactionStrategy.java
@@ -21,7 +21,7 @@ package org.apache.accumulo.test;
import java.util.Map;
import java.util.Map.Entry;
-import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.tserver.compaction.CompactionPlan;
import org.apache.accumulo.tserver.compaction.CompactionStrategy;
@@ -50,7 +50,7 @@ public class SizeCompactionStrategy extends CompactionStrategy {
public CompactionPlan getCompactionPlan(MajorCompactionRequest request) {
CompactionPlan plan = new CompactionPlan();
- for (Entry<TabletFile,DataFileValue> entry : request.getFiles().entrySet())
+ for (Entry<StoredTabletFile,DataFileValue> entry : request.getFiles().entrySet())
if (entry.getValue().getSize() < size)
plan.inputFiles.add(entry.getKey());
diff --git a/test/src/main/java/org/apache/accumulo/test/TestCompactionStrategy.java b/test/src/main/java/org/apache/accumulo/test/TestCompactionStrategy.java
index 4d3a32e..7f2eb41 100644
--- a/test/src/main/java/org/apache/accumulo/test/TestCompactionStrategy.java
+++ b/test/src/main/java/org/apache/accumulo/test/TestCompactionStrategy.java
@@ -20,6 +20,7 @@ package org.apache.accumulo.test;
import java.util.Map;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.tserver.compaction.CompactionPlan;
import org.apache.accumulo.tserver.compaction.CompactionStrategy;
@@ -60,7 +61,7 @@ public class TestCompactionStrategy extends CompactionStrategy {
public CompactionPlan getCompactionPlan(MajorCompactionRequest request) {
CompactionPlan plan = new CompactionPlan();
- for (TabletFile file : request.getFiles().keySet()) {
+ for (StoredTabletFile file : request.getFiles().keySet()) {
if (file.getFileName().startsWith(dropPrefix)) {
plan.deleteFiles.add(file);
} else if (file.getFileName().startsWith(inputPrefix)) {
diff --git a/test/src/main/java/org/apache/accumulo/test/VolumeIT.java b/test/src/main/java/org/apache/accumulo/test/VolumeIT.java
index 1bf9d6e..2a1345b 100644
--- a/test/src/main/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/VolumeIT.java
@@ -57,7 +57,7 @@ import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
-import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import org.apache.accumulo.core.security.Authorizations;
@@ -389,9 +389,9 @@ public class VolumeIT extends ConfigurableMacBase {
// check that root tablet is not on volume 1
int count = 0;
- for (TabletFile file : ((ClientContext) client).getAmple().readTablet(RootTable.EXTENT)
+ for (StoredTabletFile file : ((ClientContext) client).getAmple().readTablet(RootTable.EXTENT)
.getFiles()) {
- assertTrue(file.getMetadataEntry().startsWith(v2.toString()));
+ assertTrue(file.getMetaUpdateDelete().startsWith(v2.toString()));
count++;
}
@@ -461,10 +461,10 @@ public class VolumeIT extends ConfigurableMacBase {
// check that root tablet is not on volume 1 or 2
int count = 0;
- for (TabletFile file : ((ClientContext) client).getAmple().readTablet(RootTable.EXTENT)
+ for (StoredTabletFile file : ((ClientContext) client).getAmple().readTablet(RootTable.EXTENT)
.getFiles()) {
- assertTrue(file.getMetadataEntry().startsWith(v8.toString())
- || file.getMetadataEntry().startsWith(v9.toString()));
+ assertTrue(file.getMetaUpdateDelete().startsWith(v8.toString())
+ || file.getMetaUpdateDelete().startsWith(v9.toString()));
count++;
}
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java b/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java
index 63acca6..a4b2032 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java
@@ -439,7 +439,7 @@ public class BulkNewIT extends SharedMiniClusterBase {
for (TabletMetadata tablet : tablets) {
assertTrue(tablet.getLoaded().isEmpty());
- Set<String> fileHashes = tablet.getFiles().stream().map(f -> hash(f.getMetadataEntry()))
+ Set<String> fileHashes = tablet.getFiles().stream().map(f -> hash(f.getMetaUpdateDelete()))
.collect(Collectors.toSet());
String endRow = tablet.getEndRow() == null ? "null" : tablet.getEndRow().toString();
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java b/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
index d357d2d..83b53e8 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryIT.java
@@ -46,6 +46,7 @@ import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.file.rfile.RFile;
import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
@@ -66,6 +67,7 @@ import org.apache.accumulo.server.master.state.TServerInstance;
import org.apache.accumulo.server.util.MasterMetadataUtil;
import org.apache.accumulo.server.util.MetadataTableUtil;
import org.apache.accumulo.server.zookeeper.TransactionWatcher;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.junit.Test;
@@ -144,7 +146,7 @@ public class SplitRecoveryIT extends ConfigurableMacBase {
Text midRow = new Text(mr);
- SortedMap<TabletFile,DataFileValue> splitMapFiles = null;
+ SortedMap<StoredTabletFile,DataFileValue> splitMapFiles = null;
for (int i = 0; i < extents.length; i++) {
KeyExtent extent = extents[i];
@@ -154,16 +156,17 @@ public class SplitRecoveryIT extends ConfigurableMacBase {
ServerConstants.getTablesDirs(context)[0] + "/" + extent.getTableId() + "/" + dirName;
MetadataTableUtil.addTablet(extent, dirName, context, TimeType.LOGICAL, zl);
SortedMap<TabletFile,DataFileValue> mapFiles = new TreeMap<>();
- mapFiles.put(new TabletFile(tdir + "/" + RFile.EXTENSION + "_000_000"),
+ mapFiles.put(new TabletFile(new Path(tdir + "/" + RFile.EXTENSION + "_000_000")),
new DataFileValue(1000017 + i, 10000 + i));
- if (i == extentToSplit) {
- splitMapFiles = mapFiles;
- }
int tid = 0;
TransactionWatcher.ZooArbitrator.start(context, Constants.BULK_ARBITRATOR_TYPE, tid);
- MetadataTableUtil.updateTabletDataFile(tid, extent, mapFiles,
- new MetadataTime(0, TimeType.LOGICAL), context, zl);
+ SortedMap<StoredTabletFile,DataFileValue> storedFiles =
+ new TreeMap<>(MetadataTableUtil.updateTabletDataFile(tid, extent, mapFiles,
+ new MetadataTime(0, TimeType.LOGICAL), context, zl));
+ if (i == extentToSplit) {
+ splitMapFiles = storedFiles;
+ }
}
KeyExtent extent = extents[extentToSplit];
@@ -186,12 +189,12 @@ public class SplitRecoveryIT extends ConfigurableMacBase {
}
private void splitPartiallyAndRecover(ServerContext context, KeyExtent extent, KeyExtent high,
- KeyExtent low, double splitRatio, SortedMap<TabletFile,DataFileValue> mapFiles, Text midRow,
- String location, int steps, ZooLock zl) throws Exception {
+ KeyExtent low, double splitRatio, SortedMap<StoredTabletFile,DataFileValue> mapFiles,
+ Text midRow, String location, int steps, ZooLock zl) throws Exception {
- SortedMap<TabletFile,DataFileValue> lowDatafileSizes = new TreeMap<>();
- SortedMap<TabletFile,DataFileValue> highDatafileSizes = new TreeMap<>();
- List<TabletFile> highDatafilesToRemove = new ArrayList<>();
+ SortedMap<StoredTabletFile,DataFileValue> lowDatafileSizes = new TreeMap<>();
+ SortedMap<StoredTabletFile,DataFileValue> highDatafileSizes = new TreeMap<>();
+ List<StoredTabletFile> highDatafilesToRemove = new ArrayList<>();
MetadataTableUtil.splitDatafiles(midRow, splitRatio, new HashMap<>(), mapFiles,
lowDatafileSizes, highDatafileSizes, highDatafilesToRemove);
@@ -242,7 +245,7 @@ public class SplitRecoveryIT extends ConfigurableMacBase {
}
private void ensureTabletHasNoUnexpectedMetadataEntries(ServerContext context, KeyExtent extent,
- SortedMap<TabletFile,DataFileValue> expectedMapFiles) throws Exception {
+ SortedMap<StoredTabletFile,DataFileValue> expectedMapFiles) throws Exception {
try (Scanner scanner = new ScannerImpl(context, MetadataTable.ID, Authorizations.EMPTY)) {
scanner.setRange(extent.toMetadataRange());
@@ -297,14 +300,14 @@ public class SplitRecoveryIT extends ConfigurableMacBase {
assertTrue(sawPer);
- SortedMap<TabletFile,DataFileValue> fixedMapFiles =
+ SortedMap<StoredTabletFile,DataFileValue> fixedMapFiles =
MetadataTableUtil.getFileAndLogEntries(context, extent).getSecond();
verifySame(expectedMapFiles, fixedMapFiles);
}
}
- private void verifySame(SortedMap<TabletFile,DataFileValue> datafileSizes,
- SortedMap<TabletFile,DataFileValue> fixedDatafileSizes) throws Exception {
+ private void verifySame(SortedMap<StoredTabletFile,DataFileValue> datafileSizes,
+ SortedMap<StoredTabletFile,DataFileValue> fixedDatafileSizes) throws Exception {
if (!datafileSizes.keySet().containsAll(fixedDatafileSizes.keySet())
|| !fixedDatafileSizes.keySet().containsAll(datafileSizes.keySet())) {
@@ -312,7 +315,7 @@ public class SplitRecoveryIT extends ConfigurableMacBase {
+ fixedDatafileSizes.keySet());
}
- for (Entry<TabletFile,DataFileValue> entry : datafileSizes.entrySet()) {
+ for (Entry<StoredTabletFile,DataFileValue> entry : datafileSizes.entrySet()) {
DataFileValue dfv = entry.getValue();
DataFileValue otherDfv = fixedDatafileSizes.get(entry.getKey());
diff --git a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
index 57e3348..2bf7f55 100644
--- a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java
@@ -464,8 +464,7 @@ public class CollectTabletStats {
for (TabletFile file : files) {
FileSystem ns = fs.getVolumeByPath(file.getPath()).getFileSystem();
FileSKVIterator reader = FileOperations.getInstance().newReaderBuilder()
- .forFile(file.getMetadataEntry(), ns, ns.getConf(),
- CryptoServiceFactory.newDefaultInstance())
+ .forFile(file.getPathStr(), ns, ns.getConf(), CryptoServiceFactory.newDefaultInstance())
.withTableConfiguration(aconf).build();
Range range = new Range(ke.getPrevEndRow(), false, ke.getEndRow(), true);
reader.seek(range, columnSet, columnSet.size() != 0);
@@ -498,8 +497,7 @@ public class CollectTabletStats {
for (TabletFile file : files) {
FileSystem ns = fs.getVolumeByPath(file.getPath()).getFileSystem();
readers.add(FileOperations.getInstance().newReaderBuilder()
- .forFile(file.getMetadataEntry(), ns, ns.getConf(),
- CryptoServiceFactory.newDefaultInstance())
+ .forFile(file.getPathStr(), ns, ns.getConf(), CryptoServiceFactory.newDefaultInstance())
.withTableConfiguration(aconf.getSystemConfiguration()).build());
}