You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by cs...@apache.org on 2023/10/11 20:15:55 UTC
[accumulo] branch elasticity updated: Update AmpleConditionalWriteIT to use new file metadata format (#3833)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new e4f1f3c79f Update AmpleConditionalWriteIT to use new file metadata format (#3833)
e4f1f3c79f is described below
commit e4f1f3c79ff621727bb2f9e8d342e904e709f664
Author: Christopher L. Shannon <ch...@gmail.com>
AuthorDate: Wed Oct 11 16:15:49 2023 -0400
Update AmpleConditionalWriteIT to use new file metadata format (#3833)
After the changes with no-chop merge the StoredTabletFile metadata is
now a json string that includes a range along with the path. This change
updates the test to serialize the path into the new format with an
infinite range.
Co-authored-by: EdColeman <de...@etcoleman.com>
---
.../test/functional/AmpleConditionalWriterIT.java | 57 +++++++++++-----------
1 file changed, 29 insertions(+), 28 deletions(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java
index 5eb9078192..81883b0e2b 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java
@@ -28,6 +28,7 @@ import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType
import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW;
import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SELECTED;
import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.TIME;
+import static org.apache.accumulo.core.util.LazySingletons.GSON;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
@@ -74,6 +75,7 @@ import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.harness.AccumuloClusterHarness;
import org.apache.accumulo.server.metadata.ConditionalTabletsMutatorImpl;
import org.apache.accumulo.server.zookeeper.TransactionWatcher;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -182,14 +184,14 @@ public class AmpleConditionalWriterIT extends AccumuloClusterHarness {
var context = cluster.getServerContext();
- var stf1 = new StoredTabletFile(
- "hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000070.rf");
- var stf2 = new StoredTabletFile(
- "hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000071.rf");
- var stf3 = new StoredTabletFile(
- "hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000072.rf");
- var stf4 = new StoredTabletFile(
- "hdfs://localhost:8020/accumulo/tables/2a/default_tablet/C0000073.rf");
+ var stf1 = StoredTabletFile
+ .of(new Path("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000070.rf"));
+ var stf2 = StoredTabletFile
+ .of(new Path("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000071.rf"));
+ var stf3 = StoredTabletFile
+ .of(new Path("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000072.rf"));
+ var stf4 = StoredTabletFile
+ .of(new Path("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/C0000073.rf"));
var dfv = new DataFileValue(100, 100);
System.out.println(context.getAmple().readTablet(e1).getLocation());
@@ -279,8 +281,8 @@ public class AmpleConditionalWriterIT extends AccumuloClusterHarness {
TransactionWatcher.ZooArbitrator.start(context, Constants.BULK_ARBITRATOR_TYPE, 9L);
// simulate a bulk import
- var stf5 =
- new StoredTabletFile("hdfs://localhost:8020/accumulo/tables/2a/b-0000009/I0000074.rf");
+ var stf5 = StoredTabletFile
+ .of(new Path("hdfs://localhost:8020/accumulo/tables/2a/b-0000009/I0000074.rf"));
ctmi = new ConditionalTabletsMutatorImpl(context);
var tm6 = TabletMetadata.builder(e1).build(LOADED);
ctmi.mutateTablet(e1).requireAbsentOperation().requireSame(tm6, LOADED)
@@ -292,8 +294,8 @@ public class AmpleConditionalWriterIT extends AccumuloClusterHarness {
assertEquals(Set.of(stf4, stf5), context.getAmple().readTablet(e1).getFiles());
// simulate a compaction
- var stf6 = new StoredTabletFile(
- "hdfs://localhost:8020/accumulo/tables/2a/default_tablet/A0000075.rf");
+ var stf6 = StoredTabletFile
+ .of(new Path("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/A0000075.rf"));
ctmi = new ConditionalTabletsMutatorImpl(context);
var tm7 = TabletMetadata.builder(e1).putFile(stf4, dfv).putFile(stf5, dfv).build();
ctmi.mutateTablet(e1).requireAbsentOperation().requireSame(tm7, FILES)
@@ -320,14 +322,14 @@ public class AmpleConditionalWriterIT extends AccumuloClusterHarness {
public void testSelectedFiles() throws Exception {
var context = cluster.getServerContext();
- var stf1 =
- new StoredTabletFile("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000070.rf");
- var stf2 =
- new StoredTabletFile("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000071.rf");
- var stf3 =
- new StoredTabletFile("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000072.rf");
- var stf4 =
- new StoredTabletFile("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/C0000073.rf");
+ var stf1 = StoredTabletFile
+ .of(new Path("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000070.rf"));
+ var stf2 = StoredTabletFile
+ .of(new Path("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000071.rf"));
+ var stf3 = StoredTabletFile
+ .of(new Path("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000072.rf"));
+ var stf4 = StoredTabletFile
+ .of(new Path("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/C0000073.rf"));
var dfv = new DataFileValue(100, 100);
System.out.println(context.getAmple().readTablet(e1).getLocation());
@@ -410,9 +412,9 @@ public class AmpleConditionalWriterIT extends AccumuloClusterHarness {
var context = cluster.getServerContext();
String pathPrefix = "hdfs://localhost:8020/accumulo/tables/2a/default_tablet/";
- StoredTabletFile stf1 = new StoredTabletFile(pathPrefix + "F0000070.rf");
- StoredTabletFile stf2 = new StoredTabletFile(pathPrefix + "F0000071.rf");
- StoredTabletFile stf3 = new StoredTabletFile(pathPrefix + "F0000072.rf");
+ StoredTabletFile stf1 = StoredTabletFile.of(new Path(pathPrefix + "F0000070.rf"));
+ StoredTabletFile stf2 = StoredTabletFile.of(new Path(pathPrefix + "F0000071.rf"));
+ StoredTabletFile stf3 = StoredTabletFile.of(new Path(pathPrefix + "F0000072.rf"));
final Set<StoredTabletFile> storedTabletFiles = Set.of(stf1, stf2, stf3);
final boolean initiallySelectedAll = true;
@@ -484,7 +486,7 @@ public class AmpleConditionalWriterIT extends AccumuloClusterHarness {
TabletMetadata tm1 =
TabletMetadata.builder(e1).putSelectedFiles(selectedFiles).build(SELECTED);
ctmi = new ConditionalTabletsMutatorImpl(context);
- StoredTabletFile stf4 = new StoredTabletFile(pathPrefix + "F0000073.rf");
+ StoredTabletFile stf4 = StoredTabletFile.of(new Path(pathPrefix + "F0000073.rf"));
// submit a mutation with the condition that the selected files match what was originally
// written
DataFileValue dfv = new DataFileValue(100, 100);
@@ -516,11 +518,10 @@ public class AmpleConditionalWriterIT extends AccumuloClusterHarness {
*/
public static String createSelectedFilesJson(Long txid, boolean selAll,
Collection<String> paths) {
- String filesJsonArray =
- paths.stream().map(path -> "'" + path + "'").collect(Collectors.joining(","));
+ String filesJsonArray = GSON.get().toJson(paths);
String formattedTxid = FateTxId.formatTid(Long.parseLong(Long.toString(txid), 16));
- return ("{'txid':'" + formattedTxid + "','selAll':" + selAll + ",'files':[" + filesJsonArray
- + "]}").replace('\'', '\"');
+ return ("{'txid':'" + formattedTxid + "','selAll':" + selAll + ",'files':" + filesJsonArray
+ + "}").replace('\'', '\"');
}
@Test