You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2013/06/17 15:26:43 UTC
svn commit: r1493756 [2/2] - in /accumulo/branches/ACCUMULO-118:
core/src/main/java/org/apache/accumulo/core/
core/src/main/java/org/apache/accumulo/core/client/admin/
core/src/main/java/org/apache/accumulo/core/client/impl/
core/src/main/java/org/apac...
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java?rev=1493756&r1=1493755&r2=1493756&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/RemoveEntriesForMissingFiles.java Mon Jun 17 13:26:43 2013
@@ -66,7 +66,7 @@ public class RemoveEntriesForMissingFile
for (Entry<Key,Value> entry : metadata) {
count++;
Key key = entry.getKey();
- Path map = new Path(fs.getFullPath(key));
+ Path map = fs.getFullPath(key);
if (!fs.exists(map)) {
missing++;
log.info("File " + map + " is missing");
Modified: accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java?rev=1493756&r1=1493755&r2=1493756&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java Mon Jun 17 13:26:43 2013
@@ -16,24 +16,262 @@
*/
package org.apache.accumulo.server.util;
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
-import org.apache.accumulo.server.cli.ClientOpts;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.DefaultConfiguration;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.NumUtil;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
import com.beust.jcommander.Parameter;
public class TableDiskUsage {
+
+ private static final Logger log = Logger.getLogger(Logger.class);
+ private int nextInternalId = 0;
+ private Map<String,Integer> internalIds = new HashMap<String,Integer>();
+ private Map<Integer,String> externalIds = new HashMap<Integer,String>();
+ private Map<String,Integer[]> tableFiles = new HashMap<String,Integer[]>();
+ private Map<String,Long> fileSizes = new HashMap<String,Long>();
+
+ void addTable(String tableId) {
+ if (internalIds.containsKey(tableId))
+ throw new IllegalArgumentException("Already added table " + tableId);
+
+ int iid = nextInternalId++;
+
+ internalIds.put(tableId, iid);
+ externalIds.put(iid, tableId);
+ }
+
+ void linkFileAndTable(String tableId, String file) {
+ int internalId = internalIds.get(tableId);
+
+ Integer[] tables = tableFiles.get(file);
+ if (tables == null) {
+ tables = new Integer[internalIds.size()];
+ for (int i = 0; i < tables.length; i++)
+ tables[i] = 0;
+ tableFiles.put(file, tables);
+ }
+
+ tables[internalId] = 1;
+ }
+
+ void addFileSize(String file, long size) {
+ fileSizes.put(file, size);
+ }
+
+ Map<List<String>,Long> calculateUsage() {
+
+ Map<List<Integer>,Long> usage = new HashMap<List<Integer>,Long>();
+
+ for (Entry<String,Integer[]> entry : tableFiles.entrySet()) {
+ log.info("fileSizes " + fileSizes + " key " + Arrays.asList(entry.getKey()));
+ List<Integer> key = Arrays.asList(entry.getValue());
+ Long size = fileSizes.get(entry.getKey());
+
+ Long tablesUsage = usage.get(key);
+ if (tablesUsage == null)
+ tablesUsage = 0l;
+
+ tablesUsage += size;
+
+ usage.put(key, tablesUsage);
+
+ }
+
+ Map<List<String>,Long> externalUsage = new HashMap<List<String>,Long>();
+
+ for (Entry<List<Integer>,Long> entry : usage.entrySet()) {
+ List<String> externalKey = new ArrayList<String>();
+ List<Integer> key = entry.getKey();
+ for (int i = 0; i < key.size(); i++)
+ if (key.get(i) != 0)
+ externalKey.add(externalIds.get(i));
+
+ externalUsage.put(externalKey, entry.getValue());
+ }
+
+ return externalUsage;
+ }
+
+ public interface Printer {
+ void print(String line);
+ }
+
+ public static void printDiskUsage(AccumuloConfiguration acuConf, Collection<String> tables, FileSystem fs, Connector conn, boolean humanReadable)
+ throws TableNotFoundException,
+ IOException {
+ printDiskUsage(acuConf, tables, fs, conn, new Printer() {
+ @Override
+ public void print(String line) {
+ System.out.println(line);
+ }
+ }, humanReadable);
+ }
+
+ public static Map<TreeSet<String>,Long> getDiskUsage(AccumuloConfiguration acuConf, Set<String> tableIds, FileSystem fs, Connector conn)
+ throws IOException {
+ TableDiskUsage tdu = new TableDiskUsage();
+
+ for (String tableId : tableIds)
+ tdu.addTable(tableId);
+
+ HashSet<String> tablesReferenced = new HashSet<String>(tableIds);
+ HashSet<String> emptyTableIds = new HashSet<String>();
+
+ for (String tableId : tableIds) {
+ Scanner mdScanner = null;
+ try {
+ mdScanner = conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
+ } catch (TableNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ mdScanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+ mdScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+
+ if (!mdScanner.iterator().hasNext()) {
+ emptyTableIds.add(tableId);
+ }
+
+ for (Entry<Key,Value> entry : mdScanner) {
+ String file = entry.getKey().getColumnQualifier().toString();
+ String parts[] = file.split("/");
+ String uniqueName = parts[parts.length - 1];
+ if (file.contains(":") || file.startsWith("../")) {
+ String ref = parts[parts.length - 3];
+ if (!ref.equals(tableId)) {
+ tablesReferenced.add(ref);
+ }
+ }
+
+ tdu.linkFileAndTable(tableId, uniqueName);
+ }
+ }
+
+ for (String tableId : tablesReferenced) {
+ for (String tableDir : ServerConstants.getTablesDirs()) {
+ FileStatus[] files = fs.globStatus(new Path(tableDir + "/" + tableId + "/*/*"));
+ if (files != null) {
+ for (FileStatus fileStatus : files) {
+ // Assumes that all filenames are unique
+ String name = fileStatus.getPath().getName();
+ tdu.addFileSize(name, fileStatus.getLen());
+ }
+ }
+ }
+ }
+
+ HashMap<String,String> reverseTableIdMap = new HashMap<String,String>();
+ for (Entry<String,String> entry : conn.tableOperations().tableIdMap().entrySet())
+ reverseTableIdMap.put(entry.getValue(), entry.getKey());
+
+ TreeMap<TreeSet<String>,Long> usage = new TreeMap<TreeSet<String>,Long>(new Comparator<TreeSet<String>>() {
+
+ @Override
+ public int compare(TreeSet<String> o1, TreeSet<String> o2) {
+ int len1 = o1.size();
+ int len2 = o2.size();
+
+ int min = Math.min(len1, len2);
+
+ Iterator<String> iter1 = o1.iterator();
+ Iterator<String> iter2 = o2.iterator();
+
+ int count = 0;
+
+ while (count < min) {
+ String s1 = iter1.next();
+ String s2 = iter2.next();
+
+ int cmp = s1.compareTo(s2);
+
+ if (cmp != 0)
+ return cmp;
+
+ count++;
+ }
+
+ return len1 - len2;
+ }
+ });
+
+ for (Entry<List<String>,Long> entry : tdu.calculateUsage().entrySet()) {
+ TreeSet<String> tableNames = new TreeSet<String>();
+ for (String tableId : entry.getKey())
+ tableNames.add(reverseTableIdMap.get(tableId));
+
+ usage.put(tableNames, entry.getValue());
+ }
+
+ if (!emptyTableIds.isEmpty()) {
+ TreeSet<String> emptyTables = new TreeSet<String>();
+ for (String tableId : emptyTableIds) {
+ emptyTables.add(reverseTableIdMap.get(tableId));
+ }
+ usage.put(emptyTables, 0L);
+ }
+
+ return usage;
+ }
+
+ public static void printDiskUsage(AccumuloConfiguration acuConf, Collection<String> tables, FileSystem fs, Connector conn, Printer printer, boolean humanReadable)
+ throws TableNotFoundException, IOException {
+
+ HashSet<String> tableIds = new HashSet<String>();
+
+ for (String tableName : tables) {
+ String tableId = conn.tableOperations().tableIdMap().get(tableName);
+ if (tableId == null)
+ throw new TableNotFoundException(null, tableName, "Table " + tableName + " not found");
+
+ tableIds.add(tableId);
+ }
+
+ Map<TreeSet<String>,Long> usage = getDiskUsage(acuConf, tableIds, fs, conn);
+
+ String valueFormat = humanReadable ? "%9s" : "%,24d";
+ for (Entry<TreeSet<String>,Long> entry : usage.entrySet()) {
+ Object value = humanReadable ? NumUtil.bigNumberForSize(entry.getValue()) : entry.getValue();
+ printer.print(String.format(valueFormat + " %s", value, entry.getKey()));
+ }
+ }
+
+
static class Opts extends ClientOpts {
@Parameter(description=" <table> { <table> ... } ")
List<String> tables = new ArrayList<String>();
}
-
+
/**
* @param args
*/
@@ -42,7 +280,7 @@ public class TableDiskUsage {
Opts opts = new Opts();
opts.parseArgs(TableDiskUsage.class.getName(), args);
Connector conn = opts.getConnector();
- org.apache.accumulo.core.util.TableDiskUsage.printDiskUsage(DefaultConfiguration.getInstance(), opts.tables, fs, conn, false);
+ org.apache.accumulo.server.util.TableDiskUsage.printDiskUsage(DefaultConfiguration.getInstance(), opts.tables, fs, conn, false);
}
}
Modified: accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/MultiReaderTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/MultiReaderTest.java?rev=1493756&r1=1493755&r2=1493756&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/MultiReaderTest.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/MultiReaderTest.java Mon Jun 17 13:26:43 2013
@@ -95,7 +95,7 @@ public class MultiReaderTest {
@Test
public void testMultiReader() throws IOException {
- String manyMaps = new Path("file://" + root.getRoot().getAbsolutePath() + "/manyMaps").toString();
+ Path manyMaps = new Path("file://" + root.getRoot().getAbsolutePath() + "/manyMaps");
MultiReader reader = new MultiReader(fs, manyMaps);
IntWritable key = new IntWritable();
BytesWritable value = new BytesWritable();
Modified: accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java?rev=1493756&r1=1493755&r2=1493756&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java (original)
+++ accumulo/branches/ACCUMULO-118/server/src/test/java/org/apache/accumulo/server/tabletserver/log/SortedLogRecoveryTest.java Mon Jun 17 13:26:43 2013
@@ -118,7 +118,7 @@ public class SortedLogRecoveryTest {
final String workdir = "file://" + root.getRoot().getAbsolutePath() + "/workdir";
FileSystem fs = FileSystemImpl.getLocal();
fs.deleteRecursively(new Path(workdir));
- ArrayList<String> dirs = new ArrayList<String>();
+ ArrayList<Path> dirs = new ArrayList<Path>();
try {
for (Entry<String,KeyValue[]> entry : logs.entrySet()) {
String path = workdir + "/" + entry.getKey();
@@ -129,7 +129,7 @@ public class SortedLogRecoveryTest {
}
map.close();
ns.create(new Path(path, "finished")).close();
- dirs.add(path);
+ dirs.add(new Path(path));
}
// Recover
SortedLogRecovery recovery = new SortedLogRecovery(fs);
Modified: accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryTest.java?rev=1493756&r1=1493755&r2=1493756&view=diff
==============================================================================
--- accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryTest.java (original)
+++ accumulo/branches/ACCUMULO-118/test/src/main/java/org/apache/accumulo/test/functional/SplitRecoveryTest.java Mon Jun 17 13:26:43 2013
@@ -43,8 +43,11 @@ import org.apache.accumulo.fate.zookeepe
import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
import org.apache.accumulo.fate.zookeeper.ZooLock.LockWatcher;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
+import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.fs.FileRef;
+import org.apache.accumulo.server.fs.FileSystem;
+import org.apache.accumulo.server.fs.FileSystemImpl;
import org.apache.accumulo.server.master.state.Assignment;
import org.apache.accumulo.server.master.state.TServerInstance;
import org.apache.accumulo.server.security.SecurityConstants;
@@ -134,7 +137,7 @@ public class SplitRecoveryTest extends F
for (int i = 0; i < extents.length; i++) {
KeyExtent extent = extents[i];
- String tdir = "/dir_" + i;
+ String tdir = ServerConstants.getTablesDirs()[0] + "/" + extent.getTableId().toString() + "/dir_" + i;
MetadataTable.addTablet(extent, tdir, SecurityConstants.getSystemCredentials(), TabletTime.LOGICAL_TIME_ID, zl);
SortedMap<FileRef,DataFileValue> mapFiles = new TreeMap<FileRef,DataFileValue>();
mapFiles.put(new FileRef(tdir + "/" + RFile.EXTENSION + "_000_000"), new DataFileValue(1000017 + i, 10000 + i));