You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kr...@apache.org on 2025/04/18 14:47:08 UTC
(accumulo) branch main updated (17646f77eb -> 34416a1b67)
This is an automated email from the ASF dual-hosted git repository.
krathbun pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
from 17646f77eb Merge branch '2.1'
add aa75da3d27 Fixes bug with getting disk usage of the ROOT table (#5479)
new 34416a1b67 Merge branch '2.1'
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../accumulo/server/util/TableDiskUsage.java | 79 ++++---
.../accumulo/server/util/TableDiskUsageTest.java | 227 +++++++++++----------
2 files changed, 157 insertions(+), 149 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
Posted by kr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
krathbun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 34416a1b67c718221845745b37800f4071efd3b4
Merge: 17646f77eb aa75da3d27
Author: Kevin Rathbun <ke...@gmail.com>
AuthorDate: Fri Apr 18 10:46:10 2025 -0400
Merge branch '2.1'
.../accumulo/server/util/TableDiskUsage.java | 79 ++++---
.../accumulo/server/util/TableDiskUsageTest.java | 227 +++++++++++----------
2 files changed, 157 insertions(+), 149 deletions(-)
diff --cc server/base/src/test/java/org/apache/accumulo/server/util/TableDiskUsageTest.java
index 009fcc1cf0,5b9ee7fed2..4b71657969
--- a/server/base/src/test/java/org/apache/accumulo/server/util/TableDiskUsageTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/util/TableDiskUsageTest.java
@@@ -21,7 -21,10 +21,9 @@@ package org.apache.accumulo.server.util
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
+ import java.util.ArrayList;
-import java.util.EnumSet;
import java.util.HashMap;
+ import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@@ -32,19 -36,19 +34,15 @@@ import java.util.TreeSet
import java.util.stream.Collectors;
import org.apache.accumulo.core.Constants;
- import org.apache.accumulo.core.client.Scanner;
- import org.apache.accumulo.core.data.Key;
+ import org.apache.accumulo.core.clientImpl.ClientContext;
-import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.TableId;
--import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.metadata.AccumuloTable;
- import org.apache.accumulo.core.metadata.ReferencedTabletFile;
+ import org.apache.accumulo.core.metadata.StoredTabletFile;
+ import org.apache.accumulo.core.metadata.schema.Ample;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
--import org.apache.accumulo.core.metadata.schema.MetadataSchema;
- import org.apache.accumulo.core.security.Authorizations;
- import org.apache.accumulo.server.ServerContext;
- import org.apache.hadoop.fs.Path;
- import org.apache.hadoop.io.Text;
+ import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+ import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
-import org.apache.hadoop.io.Text;
import org.easymock.EasyMock;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
@@@ -131,25 -140,21 +134,22 @@@ public class TableDiskUsageTest
@Test
public void testMetadataTable() throws Exception {
- final ServerContext client = EasyMock.createMock(ServerContext.class);
- final Scanner scanner = EasyMock.createMock(Scanner.class);
-
- // Expect root table instead to be scanned
- EasyMock.expect(client.createScanner(AccumuloTable.ROOT.tableName(), Authorizations.EMPTY))
- .andReturn(scanner);
+ final ClientContext client = EasyMock.createMock(ClientContext.class);
EasyMock.expect(client.getTableIdToNameMap()).andReturn(tableIdToNameMap);
- final TabletsMetadata mockTabletsMetadata = mockTabletsMetadata(client, MetadataTable.ID);
++ final TabletsMetadata mockTabletsMetadata =
++ mockTabletsMetadata(client, AccumuloTable.METADATA.tableId());
- Map<Key,Value> tableEntries = new HashMap<>();
- appendFileMetadata(tableEntries, getTabletFile(AccumuloTable.METADATA.tableId(),
+ List<TabletMetadata> realTabletsMetadata = new ArrayList<>();
- appendFileMetadata(realTabletsMetadata,
- getTabletFile(volume1, MetadataTable.ID, MetadataTable.NAME, "C0001.rf"), 1024);
++ appendFileMetadata(realTabletsMetadata, getTabletFile(volume1, AccumuloTable.METADATA.tableId(),
+ AccumuloTable.METADATA.tableName(), "C0001.rf"), 1024);
- mockTableScan(scanner, tableEntries, AccumuloTable.METADATA.tableId());
+ mockTabletsMetadataIter(mockTabletsMetadata, realTabletsMetadata.iterator());
- EasyMock.replay(client, scanner);
+ EasyMock.replay(client, mockTabletsMetadata);
Map<SortedSet<String>,Long> result =
- TableDiskUsage.getDiskUsage(tableSet(MetadataTable.ID), client);
+ TableDiskUsage.getDiskUsage(tableSet(AccumuloTable.METADATA.tableId()), client);
- assertEquals(1024, getTotalUsage(result, MetadataTable.ID));
+ assertEquals(1024, getTotalUsage(result, AccumuloTable.METADATA.tableId()));
assertEquals(1, result.size());
Map.Entry<SortedSet<String>,Long> firstResult =
result.entrySet().stream().findFirst().orElseThrow();
@@@ -277,37 -294,46 +289,40 @@@
return tableIdToNameMap.get(tableId);
}
- private static void appendFileMetadata(Map<Key,Value> tableEntries, ReferencedTabletFile file,
- long size) {
- tableEntries.put(new Key(new Text(file.getTableId() + "<"),
- MetadataSchema.TabletsSection.DataFileColumnFamily.NAME, file.insert().getMetadataText()),
- new DataFileValue(size, 1).encodeAsValue());
+ private static void appendFileMetadata(List<TabletMetadata> realTabletsMetadata,
+ StoredTabletFile file, long size) {
- Key key = new Key(new Text(file.getTableId() + "<"),
- MetadataSchema.TabletsSection.DataFileColumnFamily.NAME, file.getMetaInsertText());
- Value val = new DataFileValue(size, 1).encodeAsValue();
- SortedMap<Key,Value> map = new TreeMap<>();
- map.put(key, val);
-
- TabletMetadata tm = TabletMetadata.convertRow(map.entrySet().iterator(),
- EnumSet.of(TabletMetadata.ColumnType.FILES), true);
++ TabletMetadata tm = TabletMetadata.builder(new KeyExtent(file.getTableId(), null, null))
++ .putFile(file, new DataFileValue(size, 1)).build();
+ realTabletsMetadata.add(tm);
}
- private static ReferencedTabletFile getTabletFile(String volume, TableId tableId, String tablet,
+ private static StoredTabletFile getTabletFile(String volume, TableId tableId, String tablet,
String fileName) {
- return new ReferencedTabletFile(new Path(
- return new StoredTabletFile(
- volume + Constants.HDFS_TABLES_DIR + "/" + tableId + "/" + tablet + "/" + fileName);
++ return new StoredTabletFile(StoredTabletFile.serialize(
+ volume + Constants.HDFS_TABLES_DIR + "/" + tableId + "/" + tablet + "/" + fileName));
}
- private static ReferencedTabletFile getTabletFile(TableId tableId, String tablet,
- String fileName) {
- return getTabletFile(volume1, tableId, tablet, fileName);
- }
-
- private void mockScan(ServerContext client, Scanner scanner, int times) throws Exception {
- EasyMock.expect(client.createScanner(AccumuloTable.METADATA.tableName(), Authorizations.EMPTY))
- .andReturn(scanner).times(times);
- EasyMock.expect(client.getTableIdToNameMap()).andReturn(tableIdToNameMap);
+ private TabletsMetadata mockTabletsMetadata(ClientContext client, TableId tableId) {
+ final Ample ample = EasyMock.createMock(Ample.class);
+ final TabletsMetadata.TableOptions tableOptions =
+ EasyMock.createMock(TabletsMetadata.TableOptions.class);
+ final TabletsMetadata.TableRangeOptions tableRangeOptions =
+ EasyMock.createMock(TabletsMetadata.TableRangeOptions.class);
+ final TabletsMetadata.Options options = EasyMock.createMock(TabletsMetadata.Options.class);
+ final TabletsMetadata tabletsMetadata = EasyMock.createMock(TabletsMetadata.class);
+ EasyMock.expect(client.getAmple()).andReturn(ample);
+ EasyMock.expect(ample.readTablets()).andReturn(tableOptions);
+ EasyMock.expect(tableOptions.forTable(tableId)).andReturn(tableRangeOptions);
+ EasyMock.expect(tableRangeOptions.fetch(TabletMetadata.ColumnType.FILES)).andReturn(options);
+ EasyMock.expect(options.build()).andReturn(tabletsMetadata);
+ EasyMock.replay(ample, tableOptions, tableRangeOptions, options);
+ return tabletsMetadata;
}
- private void mockTableScan(Scanner scanner, Map<Key,Value> tableEntries, TableId tableId) {
- scanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
- EasyMock.expectLastCall().once();
- scanner.setRange(new KeyExtent(tableId, null, null).toMetaRange());
- EasyMock.expectLastCall().once();
- EasyMock.expect(scanner.iterator()).andReturn(tableEntries.entrySet().iterator());
- scanner.close();
- EasyMock.expectLastCall().once();
+ private void mockTabletsMetadataIter(TabletsMetadata tabletsMetadata,
+ Iterator<TabletMetadata> realTabletsMetadata) {
+ EasyMock.expect(tabletsMetadata.iterator()).andReturn(realTabletsMetadata);
+ tabletsMetadata.close();
+ EasyMock.expectLastCall().andAnswer(() -> null);
}
}