You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2013/11/06 22:30:03 UTC
[1/2] git commit: ACCUMULO-1761 learn table directories from the
!METADATA table scan
Updated Branches:
refs/heads/master 3b9b40bb1 -> c7697db33
ACCUMULO-1761 learn table directories from the !METADATA table scan
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c7697db3
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c7697db3
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c7697db3
Branch: refs/heads/master
Commit: c7697db33a584f24d959dac7764555c94ec951d9
Parents: facca80
Author: Eric Newton <er...@gmail.com>
Authored: Wed Nov 6 16:29:52 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Wed Nov 6 16:30:15 2013 -0500
----------------------------------------------------------------------
.../java/org/apache/accumulo/server/util/TableDiskUsage.java | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7697db3/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
index b7019e6..6b80ccb 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
@@ -43,6 +43,7 @@ import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.NumUtil;
+import org.apache.accumulo.core.util.StringUtil;
import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.server.fs.VolumeManager;
@@ -147,6 +148,7 @@ public class TableDiskUsage {
HashSet<String> tablesReferenced = new HashSet<String>(tableIds);
HashSet<String> emptyTableIds = new HashSet<String>();
+ HashSet<String> nameSpacesReferenced = new HashSet<String>();
for (String tableId : tableIds) {
Scanner mdScanner = null;
@@ -171,6 +173,10 @@ public class TableDiskUsage {
if (!ref.equals(tableId)) {
tablesReferenced.add(ref);
}
+ if (file.contains(":") && parts.length > 3) {
+ List<String> base = Arrays.asList(Arrays.copyOf(parts, parts.length - 3));
+ nameSpacesReferenced.add(StringUtil.join(base, "/"));
+ }
}
tdu.linkFileAndTable(tableId, uniqueName);
@@ -178,7 +184,7 @@ public class TableDiskUsage {
}
for (String tableId : tablesReferenced) {
- for (String tableDir : ServerConstants.getTablesDirs()) {
+ for (String tableDir : nameSpacesReferenced) {
FileStatus[] files = fs.globStatus(new Path(tableDir + "/" + tableId + "/*/*"));
if (files != null) {
for (FileStatus fileStatus : files) {
[2/2] git commit: Revert "ACCUMULO-1761 use file size information
from !METADATA instead of asking the namenode,
avoids using ServerConstants.getTablesDirs()"
Posted by ec...@apache.org.
Revert "ACCUMULO-1761 use file size information from !METADATA instead of asking the namenode, avoids using ServerConstants.getTablesDirs()"
This reverts commit dcb559165b587eca9e5bf0b64df8031c748d3f37.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/facca80a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/facca80a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/facca80a
Branch: refs/heads/master
Commit: facca80a742095658ad8140467f95140451520b8
Parents: 3b9b40b
Author: Eric Newton <er...@gmail.com>
Authored: Wed Nov 6 16:17:55 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Wed Nov 6 16:30:15 2013 -0500
----------------------------------------------------------------------
.../accumulo/server/util/TableDiskUsage.java | 26 +++++++++++++-------
1 file changed, 17 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/facca80a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
index 2d87a43..b7019e6 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/TableDiskUsage.java
@@ -43,9 +43,12 @@ import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.NumUtil;
+import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.cli.ClientOpts;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
@@ -155,13 +158,11 @@ public class TableDiskUsage {
mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME);
mdScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
- Iterator<Entry<Key,Value>> mdIter = mdScanner.iterator();
- if (!mdIter.hasNext()) {
+ if (!mdScanner.iterator().hasNext()) {
emptyTableIds.add(tableId);
}
- while (mdIter.hasNext()) {
- Entry<Key,Value> entry = mdIter.next();
+ for (Entry<Key,Value> entry : mdScanner) {
String file = entry.getKey().getColumnQualifier().toString();
String parts[] = file.split("/");
String uniqueName = parts[parts.length - 1];
@@ -173,11 +174,18 @@ public class TableDiskUsage {
}
tdu.linkFileAndTable(tableId, uniqueName);
- String sizeKeys = entry.getValue().toString();
- parts = sizeKeys.split(",");
- // defensive: all file entries should have a size
- if (parts.length == 2) {
- tdu.addFileSize(uniqueName, Long.parseLong(parts[0]));
+ }
+ }
+
+ for (String tableId : tablesReferenced) {
+ for (String tableDir : ServerConstants.getTablesDirs()) {
+ FileStatus[] files = fs.globStatus(new Path(tableDir + "/" + tableId + "/*/*"));
+ if (files != null) {
+ for (FileStatus fileStatus : files) {
+ // Assumes that all filenames are unique
+ String name = fileStatus.getPath().getName();
+ tdu.addFileSize(name, fileStatus.getLen());
+ }
}
}
}