You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2013/11/07 23:39:10 UTC
git commit: ACCUMULO-1761 Remove warning introduced in prior commit
Updated Branches:
refs/heads/1.6.0-SNAPSHOT b005a2411 -> c68a23167
ACCUMULO-1761 Remove warning introduced in prior commit
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c68a2316
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c68a2316
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c68a2316
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: c68a23167738f6de5cd1d929e617a7ee079d415e
Parents: b005a24
Author: Christopher Tubbs <ct...@apache.org>
Authored: Thu Nov 7 17:38:46 2013 -0500
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Thu Nov 7 17:38:46 2013 -0500
----------------------------------------------------------------------
.../accumulo/server/util/TableDiskUsage.java | 110 +++++++++----------
1 file changed, 55 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c68a2316/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
index 6b80ccb..cb932d7 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
@@ -44,7 +44,6 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.Da
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.NumUtil;
import org.apache.accumulo.core.util.StringUtil;
-import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
@@ -56,27 +55,27 @@ import org.apache.log4j.Logger;
import com.beust.jcommander.Parameter;
public class TableDiskUsage {
-
+
private static final Logger log = Logger.getLogger(Logger.class);
private int nextInternalId = 0;
private Map<String,Integer> internalIds = new HashMap<String,Integer>();
private Map<Integer,String> externalIds = new HashMap<Integer,String>();
private Map<String,Integer[]> tableFiles = new HashMap<String,Integer[]>();
private Map<String,Long> fileSizes = new HashMap<String,Long>();
-
+
void addTable(String tableId) {
if (internalIds.containsKey(tableId))
throw new IllegalArgumentException("Already added table " + tableId);
-
+
int iid = nextInternalId++;
-
+
internalIds.put(tableId, iid);
externalIds.put(iid, tableId);
}
-
+
void linkFileAndTable(String tableId, String file) {
int internalId = internalIds.get(tableId);
-
+
Integer[] tables = tableFiles.get(file);
if (tables == null) {
tables = new Integer[internalIds.size()];
@@ -84,52 +83,52 @@ public class TableDiskUsage {
tables[i] = 0;
tableFiles.put(file, tables);
}
-
+
tables[internalId] = 1;
}
-
+
void addFileSize(String file, long size) {
fileSizes.put(file, size);
}
-
+
Map<List<String>,Long> calculateUsage() {
-
+
Map<List<Integer>,Long> usage = new HashMap<List<Integer>,Long>();
-
+
for (Entry<String,Integer[]> entry : tableFiles.entrySet()) {
log.info("fileSizes " + fileSizes + " key " + Arrays.asList(entry.getKey()));
List<Integer> key = Arrays.asList(entry.getValue());
Long size = fileSizes.get(entry.getKey());
-
+
Long tablesUsage = usage.get(key);
if (tablesUsage == null)
tablesUsage = 0l;
-
+
tablesUsage += size;
-
+
usage.put(key, tablesUsage);
-
+
}
-
+
Map<List<String>,Long> externalUsage = new HashMap<List<String>,Long>();
-
+
for (Entry<List<Integer>,Long> entry : usage.entrySet()) {
List<String> externalKey = new ArrayList<String>();
List<Integer> key = entry.getKey();
for (int i = 0; i < key.size(); i++)
if (key.get(i) != 0)
externalKey.add(externalIds.get(i));
-
+
externalUsage.put(externalKey, entry.getValue());
}
-
+
return externalUsage;
}
-
+
public interface Printer {
void print(String line);
}
-
+
public static void printDiskUsage(AccumuloConfiguration acuConf, Collection<String> tables, VolumeManager fs, Connector conn, boolean humanReadable)
throws TableNotFoundException, IOException {
printDiskUsage(acuConf, tables, fs, conn, new Printer() {
@@ -139,17 +138,18 @@ public class TableDiskUsage {
}
}, humanReadable);
}
-
- public static Map<TreeSet<String>,Long> getDiskUsage(AccumuloConfiguration acuConf, Set<String> tableIds, VolumeManager fs, Connector conn) throws IOException {
+
+ public static Map<TreeSet<String>,Long> getDiskUsage(AccumuloConfiguration acuConf, Set<String> tableIds, VolumeManager fs, Connector conn)
+ throws IOException {
TableDiskUsage tdu = new TableDiskUsage();
-
+
for (String tableId : tableIds)
tdu.addTable(tableId);
-
+
HashSet<String> tablesReferenced = new HashSet<String>(tableIds);
HashSet<String> emptyTableIds = new HashSet<String>();
HashSet<String> nameSpacesReferenced = new HashSet<String>();
-
+
for (String tableId : tableIds) {
Scanner mdScanner = null;
try {
@@ -159,11 +159,11 @@ public class TableDiskUsage {
}
mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
mdScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
-
+
if (!mdScanner.iterator().hasNext()) {
emptyTableIds.add(tableId);
}
-
+
for (Entry<Key,Value> entry : mdScanner) {
String file = entry.getKey().getColumnQualifier().toString();
String parts[] = file.split("/");
@@ -178,11 +178,11 @@ public class TableDiskUsage {
nameSpacesReferenced.add(StringUtil.join(base, "/"));
}
}
-
+
tdu.linkFileAndTable(tableId, uniqueName);
}
}
-
+
for (String tableId : tablesReferenced) {
for (String tableDir : nameSpacesReferenced) {
FileStatus[] files = fs.globStatus(new Path(tableDir + "/" + tableId + "/*/*"));
@@ -195,49 +195,49 @@ public class TableDiskUsage {
}
}
}
-
+
HashMap<String,String> reverseTableIdMap = new HashMap<String,String>();
for (Entry<String,String> entry : conn.tableOperations().tableIdMap().entrySet())
reverseTableIdMap.put(entry.getValue(), entry.getKey());
-
+
TreeMap<TreeSet<String>,Long> usage = new TreeMap<TreeSet<String>,Long>(new Comparator<TreeSet<String>>() {
-
+
@Override
public int compare(TreeSet<String> o1, TreeSet<String> o2) {
int len1 = o1.size();
int len2 = o2.size();
-
+
int min = Math.min(len1, len2);
-
+
Iterator<String> iter1 = o1.iterator();
Iterator<String> iter2 = o2.iterator();
-
+
int count = 0;
-
+
while (count < min) {
String s1 = iter1.next();
String s2 = iter2.next();
-
+
int cmp = s1.compareTo(s2);
-
+
if (cmp != 0)
return cmp;
-
+
count++;
}
-
+
return len1 - len2;
}
});
-
+
for (Entry<List<String>,Long> entry : tdu.calculateUsage().entrySet()) {
TreeSet<String> tableNames = new TreeSet<String>();
for (String tableId : entry.getKey())
tableNames.add(reverseTableIdMap.get(tableId));
-
+
usage.put(tableNames, entry.getValue());
}
-
+
if (!emptyTableIds.isEmpty()) {
TreeSet<String> emptyTables = new TreeSet<String>();
for (String tableId : emptyTableIds) {
@@ -245,37 +245,37 @@ public class TableDiskUsage {
}
usage.put(emptyTables, 0L);
}
-
+
return usage;
}
-
+
public static void printDiskUsage(AccumuloConfiguration acuConf, Collection<String> tables, VolumeManager fs, Connector conn, Printer printer,
boolean humanReadable) throws TableNotFoundException, IOException {
-
+
HashSet<String> tableIds = new HashSet<String>();
-
+
for (String tableName : tables) {
String tableId = conn.tableOperations().tableIdMap().get(tableName);
if (tableId == null)
throw new TableNotFoundException(null, tableName, "Table " + tableName + " not found");
-
+
tableIds.add(tableId);
}
-
+
Map<TreeSet<String>,Long> usage = getDiskUsage(acuConf, tableIds, fs, conn);
-
+
String valueFormat = humanReadable ? "%9s" : "%,24d";
for (Entry<TreeSet<String>,Long> entry : usage.entrySet()) {
Object value = humanReadable ? NumUtil.bigNumberForSize(entry.getValue()) : entry.getValue();
printer.print(String.format(valueFormat + " %s", value, entry.getKey()));
}
}
-
+
static class Opts extends ClientOpts {
@Parameter(description = " <table> { <table> ... } ")
List<String> tables = new ArrayList<String>();
}
-
+
/**
* @param args
*/
@@ -286,5 +286,5 @@ public class TableDiskUsage {
Connector conn = opts.getConnector();
org.apache.accumulo.server.util.TableDiskUsage.printDiskUsage(DefaultConfiguration.getInstance(), opts.tables, fs, conn, false);
}
-
+
}